From 97fe7ffe0ba355ce2782d7aaf688bbcec3eb416f Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 22 Feb 2016 09:46:39 -0600 Subject: [PATCH 001/507] Use variables for i2b2 meta/data schemas. - SQL Error: ORA-00942: table or view does not exist (loyalty_cohort_patient_summary) - Create an empty table for now --- Oracle/PCORNetLoader_ora.sql | 38 ++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index e555016..be33303 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -12,6 +12,20 @@ -- (please see the original MSSQL version script.) ------------------------------------------------------------------------------------------- +--For undefining data/meta schema variables (SQLDeveloper at least) +--undef i2b2_data_schema; +--undef i2b2_meta_schema; + +/* Create the loyalty cohort summary table if it doesn't exist. +TODO: Determine the side effects of doing so - I don't quite know what this +table means or how having an empty one affects the results of the transform. +*/ +whenever sqlerror continue; +create table "&&i2b2_data_schema".loyalty_cohort_patient_summary ( + patient_num number, + filter_set number +); +whenever sqlerror exit; create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS BEGIN @@ -55,7 +69,7 @@ END PMN_ExecuateSQL; -CREATE OR REPLACE SYNONYM I2B2FACT FOR I2B2DEMODATA.OBSERVATION_FACT +CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT / @@ -71,32 +85,32 @@ select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('01-Jan-201 ) where ROWNUM<100000000 / -create or replace VIEW i2b2patient as select * from I2B2DEMODATA.PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) +create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) / -create or replace view i2b2visit as select * from I2B2DEMODATA.VISIT_DIMENSION where START_DATE >= to_date('01-Jan-2010','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE = to_date('01-Jan-2010','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE Date: Tue, 23 Feb 2016 14:00:32 -0600 Subject: [PATCH 002/507] - Stubbed out/added TODOs regarding missing columns (discharge_status, facility_id, etc.) - Some of these are columns we need to add I believe. - Commented out/added TODOs about compilation errors regarding missing tables. - These tables appear to be created in the function itself? - Make datamart_id, datamart_name into variables. - Added undef comment hints. - Remove loyalty cohort table creation - this is done with code in the SCILHS-utils repository. - For now, allow all patients into the i2b2_loyalty_patients view. - Added TODOs/Oracle error messages regarding function compiler errors. - Commented out functions in pcornetloader not yet working due to compile errors, etc. --- Oracle/PCORNetLoader_ora.sql | 119 +++++++++++++++++++++++++++++------ 1 file changed, 101 insertions(+), 18 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index be33303..92d4a1f 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -115,13 +115,13 @@ CREATE OR REPLACE SYNONYM pcornet_enc FOR "&&i2b2_meta_schema".pcornet_enc create or replace FUNCTION GETDATAMARTID RETURN VARCHAR2 IS BEGIN - RETURN 'WF'; + RETURN &&datamart_id; END; / CREATE OR REPLACE FUNCTION GETDATAMARTNAME RETURN VARCHAR2 AS BEGIN - RETURN 'WakeForest'; + RETURN &&datamart_name; END; / @@ -131,8 +131,17 @@ BEGIN END; / -create or replace view i2b2loyalty_patients as (select patient_num,to_date('01-Jul-2010','dd-mon-rrrr') period_start,to_date('01-Jul-2014','dd-mon-rrrr') period_end from "&&i2b2_data_schema".loyalty_cohort_patient_summary where BITAND(filter_set, 61511) = 61511 and patient_num in (select patient_num from i2b2patient)) -/ +--create or replace view i2b2loyalty_patients as (select patient_num,to_date('01-Jul-2010','dd-mon-rrrr') period_start,to_date('01-Jul-2014','dd-mon-rrrr') period_end from "&&i2b2_data_schema".loyalty_cohort_patient_summary where BITAND(filter_set, 61511) = 61511 and patient_num in (select patient_num from i2b2patient)) +--/ + +--For testing on the tiny KUMC test set, let's count all patients +create or replace view i2b2loyalty_patients as ( + select + patient_num, to_date('01-Jul-2010','dd-mon-rrrr') period_start, + to_date('01-Jul-2014','dd-mon-rrrr') period_end + from "&&i2b2_data_schema".patient_dimension + ); + BEGIN PMN_DROPSQL('DROP TABLE pcornet_codelist'); @@ -941,6 +950,36 @@ end PCORNetDemographic; +/* TODOs: + +1) +Figure out what "_source" is supposed to be. I'ts an invalid name in +oracle as-is. And, I don't see where it comes from - the tables were selecting +from don't have it. + +Error(16,266): PL/SQL: ORA-00911: invalid character +00911. 00000 - "invalid character" +*Cause: identifiers may not start with any ASCII character other than + letters and numbers. $#_ are also allowed after the first + character. + +Commented out that and what it's used for (ADMITTING_SOURCE) + +2) +ORA-00904: "DISCHARGE_STATUS": invalid identifier + +3) +ORA-00904: "DISCHARGE_DISPOSITION": invalid identifier + +4) +ORA-00904: "FACILITY_ID": invalid identifier + +5) +ORA-00904: "LOCATION_ZIP": invalid identifier + +6) +ORA-00904: "PROVIDERID": invalid identifier +*/ create or replace procedure PCORNetEncounter as sqltext varchar2(4000); @@ -955,8 +994,13 @@ select distinct v.patient_num, v.encounter_num, to_char(start_Date,'HH:MI'), end_Date, to_char(end_Date,'HH:MI'), - providerid,location_zip, -(case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, facility_id, CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END , CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END , drg.drg, drg_type, CASE WHEN _source IS NULL THEN 'NI' ELSE admitting_source END + 'NI' providerid, --See TODO above + 'NI' location_zip, /* See TODO above */ +(case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, + 'NI' facility_id, /* See TODO above */ + 'NI' discharge_disposition /*CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END*/ , + 'NI' discharge_status /* See TODO above CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END*/ , + drg.drg, drg_type, 'NI' admitting_source /* see TODO above CASE WHEN _source IS NULL THEN 'NI' ELSE admitting_source END*/ from i2b2visit v inner join pmndemographic d on v.patient_num=d.patid left outer join (select * from @@ -1199,7 +1243,10 @@ INSERT INTO pmnENROLLMENT(PATID, ENR_START_DATE, ENR_END_DATE, CHART, BASIS) end PCORNetEnroll; / - +/* TODO: why? Error(106,17): PL/SQL: ORA-00942: table or view does not exist +Well, that's a compiler error and the function itself is supposed to create that +table. So, maybe that's ok? Perhaps it'll work at runtime? +*/ create or replace procedure PCORNetLabResultCM as sqltext varchar2(4000); begin @@ -1258,7 +1305,7 @@ INSERT INTO pmnlabresults_cm ,RAW_ORDER_DEPT ,RAW_FACILITY_CODE) - +--00942. 00000 - "table or view does not exist" Error at Line: 1,342 Column: 17 (location) SELECT DISTINCT M.patient_num patid, M.encounter_num encounterid, CASE WHEN ont_parent.C_BASECODE LIKE 'LAB_NAME%' then SUBSTR (ont_parent.c_basecode,10, 10) ELSE 'NI' END LAB_NAME, @@ -1336,7 +1383,10 @@ end PCORNetHarvest; - +/* TODO: Error(93,15): PL/SQL: ORA-00942: table or view does not exist +At compile time, it's complaining about the fact tables don't exist that are +created in the function itself. +*/ create or replace procedure PCORNetPrescribing as sqltext varchar2(4000); begin @@ -1439,7 +1489,7 @@ end PCORNetPrescribing; / - +-- TODO: Same compile-time errors about tables created in the function create or replace procedure PCORNetDispensing as sqltext varchar2(4000); begin @@ -1569,21 +1619,34 @@ end pcornetReport; - +/* These have compile errors at least partly due to the compile errors noted above +regarding missing tables that are created inside the function itself. +Error(12,1): PLS-00905: object NGRAHAM.PCORNETLABRESULTCM is invalid +Error(12,1): PL/SQL: Statement ignored +Error(13,1): PLS-00905: object NGRAHAM.PCORNETPRESCRIBING is invalid +Error(13,1): PL/SQL: Statement ignored +Error(14,1): PLS-00905: object NGRAHAM.PCORNETDISPENSING is invalid +Error(14,1): PL/SQL: Statement ignored +*/ create or replace procedure pcornetloader as begin ---pcornetclear; PCORNetHarvest; PCORNetDemographic; PCORNetEncounter; -PCORNetDiagnosis; -PCORNetCondition; -PCORNetProcedure; -PCORNetVital; +--PCORNetDiagnosis; +--PCORNetCondition; + +/* Commented out PCORNetProcedure, PCORNetVital as they take a _very_ long time +even with our tiny test set. Cartesian joins with the fact table. */ +--PCORNetProcedure; +--PCORNetVital; PCORNetEnroll; -PCORNetLabResultCM; -PCORNetPrescribing; -PCORNetDispensing; + +-- Comment out due to compile errors noted above +--PCORNetLabResultCM; +--PCORNetPrescribing; +--PCORNetDispensing; end pcornetloader; / @@ -1593,6 +1656,26 @@ end pcornetloader; + +/* TODO: PCORNetDiagnosis, PCORNetCondition causes the following for some reason. +Perhaps because I'm creating the function in my own schema and it's referencing +(and possibly creating) things in another. + +01031. 00000 - "insufficient privileges" +*Cause: An attempt was made to change the current username or password + without the appropriate privilege. This error also occurs if + attempting to install a database without the necessary operating + system privileges. + When Trusted Oracle is configure in DBMS MAC, this error may occur + if the user was granted the necessary privilege at a higher label + than the current login. +*Action: Ask the database administrator to perform the operation or grant + the required privileges. + For Trusted Oracle users getting this error although granted the + the appropriate privilege at a higher label, ask the database + administrator to regrant the privilege at the appropriate label. + +*/ BEGIN pcornetloader; --- you may want to run sql statements one by one in the pcornetloader procedure :) END; From eab63d3ba8c38b51811697c61afcfcae4e88d79f Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 23 Feb 2016 16:31:31 -0600 Subject: [PATCH 003/507] Added comment/TODOabout race vs. ethnicity. --- Oracle/PCORNetLoader_ora.sql | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 92d4a1f..781f996 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -797,9 +797,24 @@ END; / - - - +/* TODO: Race vs Ethnicity - I think Eric at MCRF switched things around to use +the fact table rather than the RACE_CD in the patient dimension. +Refs: +https://informatics.gpcnetwork.org/trac/Project/ticket/191#comment:15 +http://listserv.kumc.edu/pipermail/gpc-dev/2016q1/002508.html (thread) + +Maybe something like the following, along with a change to the demographic +procedure? + + +update blueheronmetadata.pcornet_demo set + c_facttablecolumn = 'CONCEPT_CD', + c_tablename='CONCEPT_DIMENSION', + c_columnname = 'CONCEPT_PATH', + c_operator='LIKE', + c_dimcode='\i2b2\Demographics\Ethnicity\Hispanic' +where c_fullname = '\PCORI\DEMOGRAPHIC\HISPANIC\Y\'; +*/ create or replace procedure PCORNetDemographic as From c2172ef4fb57f6c1033f543c11d42c38e42f9fee Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 24 Feb 2016 11:27:00 -0600 Subject: [PATCH 004/507] Permissions errors were resolved with grants to allow create/drop tables within procedures - grant create table to... - grant drop any table to --- Oracle/PCORNetLoader_ora.sql | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 781f996..bc45360 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1649,8 +1649,8 @@ begin PCORNetHarvest; PCORNetDemographic; PCORNetEncounter; ---PCORNetDiagnosis; ---PCORNetCondition; +PCORNetDiagnosis; +PCORNetCondition; /* Commented out PCORNetProcedure, PCORNetVital as they take a _very_ long time even with our tiny test set. Cartesian joins with the fact table. */ @@ -1667,30 +1667,6 @@ end pcornetloader; / - - - - - -/* TODO: PCORNetDiagnosis, PCORNetCondition causes the following for some reason. -Perhaps because I'm creating the function in my own schema and it's referencing -(and possibly creating) things in another. - -01031. 00000 - "insufficient privileges" -*Cause: An attempt was made to change the current username or password - without the appropriate privilege. This error also occurs if - attempting to install a database without the necessary operating - system privileges. - When Trusted Oracle is configure in DBMS MAC, this error may occur - if the user was granted the necessary privilege at a higher label - than the current login. -*Action: Ask the database administrator to perform the operation or grant - the required privileges. - For Trusted Oracle users getting this error although granted the - the appropriate privilege at a higher label, ask the database - administrator to regrant the privilege at the appropriate label. - -*/ BEGIN pcornetloader; --- you may want to run sql statements one by one in the pcornetloader procedure :) END; From 49b30940995c5e3647583e65dbda365c1471cdec Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 24 Feb 2016 12:31:26 -0600 Subject: [PATCH 005/507] Added notes/TODOs on fixing compilation errors with the PCORNetLabResultCM procedure. --- Oracle/PCORNetLoader_ora.sql | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index bc45360..2cf4d95 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1257,11 +1257,36 @@ INSERT INTO pmnENROLLMENT(PATID, ENR_START_DATE, ENR_END_DATE, CHART, BASIS) end PCORNetEnroll; / +/* TODO: Review this: I got Error(106,17): PL/SQL: ORA-00942: table or view does +not exist apparently due to tables that the procedure references but also drops/ +recreates before reference. Creating them outside the function solves the issue: + +create table priority as ( + select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode PRIORITY + from i2b2fact + inner join pmnENCOUNTER enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num + inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode + where c_fullname LIKE '\PCORI_MOD\PRIORITY\%' + ); + +create table location as ( + select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode RESULT_LOC + from i2b2fact + inner join pmnENCOUNTER enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num + inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode + where c_fullname LIKE '\PCORI_MOD\RESULT_LOC\%' +); +*/ -/* TODO: why? Error(106,17): PL/SQL: ORA-00942: table or view does not exist -Well, that's a compiler error and the function itself is supposed to create that -table. So, maybe that's ok? Perhaps it'll work at runtime? +/* TODO: Review: I got Error(63,123): PL/SQL: ORA-00904: "LAB"."PCORI_SPECIMEN_SOURCE": invalid identifier + +So, I just altered the table to have the referenced column. + +alter table blueheronmetadata.pcornet_lab add ( + pcori_specimen_source varchar2(1000) -- arbitrary + ); */ + create or replace procedure PCORNetLabResultCM as sqltext varchar2(4000); begin From 3e48dca3d4d1dade660fc897b0b2de67b40e1817 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Thu, 25 Feb 2016 15:52:14 -0600 Subject: [PATCH 006/507] Added notes/commented SQL to create missing tables to functions would compile. - Added notes/error messages/commented SQL to add missing columns. - Uncommented lab/medication functions since they now compile. --- Oracle/PCORNetLoader_ora.sql | 78 +++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 2cf4d95..882e9ed 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1425,8 +1425,55 @@ end PCORNetHarvest; /* TODO: Error(93,15): PL/SQL: ORA-00942: table or view does not exist At compile time, it's complaining about the fact tables don't exist that are -created in the function itself. +created in the function itself. I created them ahead of time with the following: + +create table basis as ( + select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2fact basis + inner join pmnENCOUNTER enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num + join pcornet_med basiscode + on basis.modifier_cd = basiscode.c_basecode + and basiscode.c_fullname like '\PCORI_MOD\RX_BASIS\%' + ); + +create table freq as ( + select pcori_basecode,encounter_num,concept_cd from i2b2fact freq + inner join pmnENCOUNTER enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num + join pcornet_med freqcode + on freq.modifier_cd = freqcode.c_basecode + and freqcode.c_fullname like '\PCORI_MOD\RX_FREQUENCY\%' + ); + +create table quantity as ( + select nval_num,encounter_num,concept_cd from i2b2fact quantity + inner join pmnENCOUNTER enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num + join pcornet_med quantitycode + on quantity.modifier_cd = quantitycode.c_basecode + and quantitycode.c_fullname like '\PCORI_MOD\RX_QUANTITY\' + ); + +create table refills as ( + select nval_num,encounter_num,concept_cd from i2b2fact refills + inner join pmnENCOUNTER enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num + join pcornet_med refillscode + on refills.modifier_cd = refillscode.c_basecode + and refillscode.c_fullname like '\PCORI_MOD\RX_REFILLS\' + ); + +create table supply as ( + select nval_num,encounter_num,concept_cd from i2b2fact supply + inner join pmnENCOUNTER enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num + join pcornet_med supplycode + on supply.modifier_cd = supplycode.c_basecode + and supplycode.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\' + ); + +TODO: Also: Error(71,159): PL/SQL: ORA-00904: "MO"."PCORI_CUI": invalid identifier +alter table blueheronmetadata.pcornet_med add ( + pcori_cui varchar2(1000) -- arbitrary + ); + */ + create or replace procedure PCORNetPrescribing as sqltext varchar2(4000); begin @@ -1529,7 +1576,28 @@ end PCORNetPrescribing; / + -- TODO: Same compile-time errors about tables created in the function +/* +Error(53,16): PL/SQL: ORA-00942: table or view does not exist (amount) + - supply is created above in the prescribing function + +create table amount as ( + select nval_num,encounter_num,concept_cd from i2b2fact amount + join pcornet_med amountcode + on amount.modifier_cd = amountcode.c_basecode + and amountcode.c_fullname like '\PCORI_MOD\RX_QUANTITY\' + ); + +TODO: Error(57,57): PL/SQL: ORA-00904: "MO"."PCORI_NDC": invalid identifier + +alter table blueheronmetadata.pcornet_med add ( + pcori_ndc varchar2(1000) -- arbitrary + ); + +\pcornet_med +*/ + create or replace procedure PCORNetDispensing as sqltext varchar2(4000); begin @@ -1682,11 +1750,9 @@ even with our tiny test set. Cartesian joins with the fact table. */ --PCORNetProcedure; --PCORNetVital; PCORNetEnroll; - --- Comment out due to compile errors noted above ---PCORNetLabResultCM; ---PCORNetPrescribing; ---PCORNetDispensing; +PCORNetLabResultCM; +PCORNetPrescribing; +PCORNetDispensing; end pcornetloader; / From 4ff4e322ae7fc200a9b2998f9ce508d85dcadf68 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Thu, 25 Feb 2016 17:55:56 -0600 Subject: [PATCH 007/507] Uncomment Procedure/Vital - they completed in 10min/5min respectively. --- Oracle/PCORNetLoader_ora.sql | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 882e9ed..b6741d3 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1744,11 +1744,8 @@ PCORNetDemographic; PCORNetEncounter; PCORNetDiagnosis; PCORNetCondition; - -/* Commented out PCORNetProcedure, PCORNetVital as they take a _very_ long time -even with our tiny test set. Cartesian joins with the fact table. */ ---PCORNetProcedure; ---PCORNetVital; +PCORNetProcedure; +PCORNetVital; PCORNetEnroll; PCORNetLabResultCM; PCORNetPrescribing; From 6f0cdcb960e86ba6624ecf804150e49b9db28a02 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Fri, 26 Feb 2016 09:50:55 -0600 Subject: [PATCH 008/507] Uncomment table creation workarounds - surround with whenever "sqlerror continue" for running with sqlplus. - Remove obsolete loyalty cohort workaround - Add comments clarifying motivation for workarounds - Comment out PCORNEtDispensing with justification comment --- Oracle/PCORNetLoader_ora.sql | 118 +++++++++++++++++++++-------------- 1 file changed, 70 insertions(+), 48 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index b6741d3..d0bc4b4 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -16,16 +16,6 @@ --undef i2b2_data_schema; --undef i2b2_meta_schema; -/* Create the loyalty cohort summary table if it doesn't exist. -TODO: Determine the side effects of doing so - I don't quite know what this -table means or how having an empty one affects the results of the transform. -*/ -whenever sqlerror continue; -create table "&&i2b2_data_schema".loyalty_cohort_patient_summary ( - patient_num number, - filter_set number -); -whenever sqlerror exit; create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS BEGIN @@ -131,10 +121,15 @@ BEGIN END; / +/* TODO: Consider building the loyalty cohort as designed: +https://github.com/njgraham/SCILHS-utils/blob/master/LoyaltyCohort/LoyaltyCohort-ora.sql + +For now, let's count all patients for testing with the KUMC test patients. +*/ + --create or replace view i2b2loyalty_patients as (select patient_num,to_date('01-Jul-2010','dd-mon-rrrr') period_start,to_date('01-Jul-2014','dd-mon-rrrr') period_end from "&&i2b2_data_schema".loyalty_cohort_patient_summary where BITAND(filter_set, 61511) = 61511 and patient_num in (select patient_num from i2b2patient)) --/ ---For testing on the tiny KUMC test set, let's count all patients create or replace view i2b2loyalty_patients as ( select patient_num, to_date('01-Jul-2010','dd-mon-rrrr') period_start, @@ -1257,9 +1252,21 @@ INSERT INTO pmnENROLLMENT(PATID, ENR_START_DATE, ENR_END_DATE, CHART, BASIS) end PCORNetEnroll; / -/* TODO: Review this: I got Error(106,17): PL/SQL: ORA-00942: table or view does -not exist apparently due to tables that the procedure references but also drops/ -recreates before reference. Creating them outside the function solves the issue: + +/* TODO: When compiling PCORNetLabResultCM I got Error(106,17): + PL/SQL: ORA-00942: table or view does not exist +apparently due to tables that the procedure references but also drops/recreates +before reference. Creating them outside the function solves the issue. SQL +copied from the function. + +In the same function, I got + Error(63,123): PL/SQL: ORA-00904: "LAB"."PCORI_SPECIMEN_SOURCE": invalid identifier +So, I just altered the table to have the referenced column. + +*/ +whenever sqlerror continue; +drop table priority; +drop table location; create table priority as ( select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode PRIORITY @@ -1276,16 +1283,11 @@ create table location as ( inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode where c_fullname LIKE '\PCORI_MOD\RESULT_LOC\%' ); -*/ - -/* TODO: Review: I got Error(63,123): PL/SQL: ORA-00904: "LAB"."PCORI_SPECIMEN_SOURCE": invalid identifier - -So, I just altered the table to have the referenced column. alter table blueheronmetadata.pcornet_lab add ( pcori_specimen_source varchar2(1000) -- arbitrary ); -*/ +whenever sqlerror exit; create or replace procedure PCORNetLabResultCM as sqltext varchar2(4000); @@ -1345,7 +1347,6 @@ INSERT INTO pmnlabresults_cm ,RAW_ORDER_DEPT ,RAW_FACILITY_CODE) ---00942. 00000 - "table or view does not exist" Error at Line: 1,342 Column: 17 (location) SELECT DISTINCT M.patient_num patid, M.encounter_num encounterid, CASE WHEN ont_parent.C_BASECODE LIKE 'LAB_NAME%' then SUBSTR (ont_parent.c_basecode,10, 10) ELSE 'NI' END LAB_NAME, @@ -1414,18 +1415,27 @@ create or replace procedure PCORNetHarvest as begin INSERT INTO pmnharvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMENT_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) - VALUES('SCILHS', 'SCILHS', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, 01, 02, 1,1,2,1,2,1,2,1,2,1,1,2,2,1,1,1,2,1,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,null,current_date,null,null,null); + VALUES(&&network_id, &&network_name, getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, 01, 02, 1,1,2,1,2,1,2,1,2,1,1,2,2,1,1,1,2,1,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,null,current_date,null,null,null); end PCORNetHarvest; / - - -/* TODO: Error(93,15): PL/SQL: ORA-00942: table or view does not exist +/* TODO: When compiling PCORNetPrescribing, I got Error(93,15): + PL/SQL: ORA-00942: table or view does not exist At compile time, it's complaining about the fact tables don't exist that are -created in the function itself. I created them ahead of time with the following: +created in the function itself. I created them ahead of time - SQL taken from +the procedure. + +Also: Error(71,159): PL/SQL: ORA-00904: "MO"."PCORI_CUI": invalid identifier +*/ +whenever sqlerror continue; +drop table basis; +drop table freq; +drop table quantity; +drop table refills; +drop table supply; create table basis as ( select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2fact basis @@ -1466,13 +1476,11 @@ create table supply as ( on supply.modifier_cd = supplycode.c_basecode and supplycode.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\' ); - -TODO: Also: Error(71,159): PL/SQL: ORA-00904: "MO"."PCORI_CUI": invalid identifier + alter table blueheronmetadata.pcornet_med add ( pcori_cui varchar2(1000) -- arbitrary ); - -*/ +whenever sqlerror exit; create or replace procedure PCORNetPrescribing as sqltext varchar2(4000); @@ -1577,10 +1585,15 @@ end PCORNetPrescribing; --- TODO: Same compile-time errors about tables created in the function -/* +/* TODO: When compiling PCORNetDispensing: + Error(53,16): PL/SQL: ORA-00942: table or view does not exist (amount) - - supply is created above in the prescribing function + - supply, also used, is created above in the prescribing function + +Also, Error(57,57): PL/SQL: ORA-00904: "MO"."PCORI_NDC": invalid identifier +*/ +whenever sqlerror continue; +drop table amount; create table amount as ( select nval_num,encounter_num,concept_cd from i2b2fact amount @@ -1588,15 +1601,11 @@ create table amount as ( on amount.modifier_cd = amountcode.c_basecode and amountcode.c_fullname like '\PCORI_MOD\RX_QUANTITY\' ); - -TODO: Error(57,57): PL/SQL: ORA-00904: "MO"."PCORI_NDC": invalid identifier alter table blueheronmetadata.pcornet_med add ( pcori_ndc varchar2(1000) -- arbitrary ); - -\pcornet_med -*/ +whenever sqlerror exit; create or replace procedure PCORNetDispensing as sqltext varchar2(4000); @@ -1727,15 +1736,7 @@ end pcornetReport; -/* These have compile errors at least partly due to the compile errors noted above -regarding missing tables that are created inside the function itself. -Error(12,1): PLS-00905: object NGRAHAM.PCORNETLABRESULTCM is invalid -Error(12,1): PL/SQL: Statement ignored -Error(13,1): PLS-00905: object NGRAHAM.PCORNETPRESCRIBING is invalid -Error(13,1): PL/SQL: Statement ignored -Error(14,1): PLS-00905: object NGRAHAM.PCORNETDISPENSING is invalid -Error(14,1): PL/SQL: Statement ignored -*/ + create or replace procedure pcornetloader as begin ---pcornetclear; @@ -1749,7 +1750,28 @@ PCORNetVital; PCORNetEnroll; PCORNetLabResultCM; PCORNetPrescribing; -PCORNetDispensing; + +/* ORA-04068: existing state of packages has been discarded +ORA-04065: not executed, altered or dropped stored procedure "NGRAHAM.PCORNETDISPENSING" +ORA-06508: PL/SQL: could not find program unit being called: "NGRAHAM.PCORNETDISPENSING" +ORA-06512: at "NGRAHAM.PCORNETLOADER", line 14 +ORA-06512: at line 2 +04068. 00000 - "existing state of packages%s%s%s has been discarded" +*Cause: One of errors 4060 - 4067 when attempt to execute a stored + procedure. +*Action: Try again after proper re-initialization of any application's + state. + +The above error only happens when we call PCORNetDispensing _and_ PCORNetPrescribing +from within pcornetloader. When running either individually, the error does not +happen. + +Skipping dispensing as per gpc-dev notes: +http://listserv.kumc.edu/pipermail/gpc-dev/attachments/20160223/8d79fa70/attachment-0001.pdf +> LV: the dispensing side [?] is not mandatory? we just did Rx, since that +> what we have in our i2b2 +*/ +--PCORNetDispensing; end pcornetloader; / From 8b0ec531731ee6ac706082ed29bf743cbec18244 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Fri, 26 Feb 2016 11:02:55 -0600 Subject: [PATCH 009/507] Quote string variables. - Added undef hints for datamart/network variables. --- Oracle/PCORNetLoader_ora.sql | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index d0bc4b4..880ca1b 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -15,6 +15,10 @@ --For undefining data/meta schema variables (SQLDeveloper at least) --undef i2b2_data_schema; --undef i2b2_meta_schema; +--undef datamart_id; +--undef datamart_name; +--undef network_id; +--undef network_name; create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS @@ -105,13 +109,13 @@ CREATE OR REPLACE SYNONYM pcornet_enc FOR "&&i2b2_meta_schema".pcornet_enc create or replace FUNCTION GETDATAMARTID RETURN VARCHAR2 IS BEGIN - RETURN &&datamart_id; + RETURN '&&datamart_id'; END; / CREATE OR REPLACE FUNCTION GETDATAMARTNAME RETURN VARCHAR2 AS BEGIN - RETURN &&datamart_name; + RETURN '&&datamart_name'; END; / @@ -1415,7 +1419,7 @@ create or replace procedure PCORNetHarvest as begin INSERT INTO pmnharvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMENT_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) - VALUES(&&network_id, &&network_name, getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, 01, 02, 1,1,2,1,2,1,2,1,2,1,1,2,2,1,1,1,2,1,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,null,current_date,null,null,null); + VALUES('&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, 01, 02, 1,1,2,1,2,1,2,1,2,1,1,2,2,1,1,1,2,1,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,null,current_date,null,null,null); end PCORNetHarvest; / From abdb2879495a549a0d1ee537008620b62b75a13e Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Fri, 26 Feb 2016 13:49:55 -0600 Subject: [PATCH 010/507] Renamed tables according to Common Data Model (CDM) Specification, Version 3.0 --- Oracle/PCORNetLoader_ora.sql | 278 +++++++++++++++++------------------ 1 file changed, 139 insertions(+), 139 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 880ca1b..edb560f 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -220,11 +220,11 @@ end pcornet_popcodelist; BEGIN -PMN_DROPSQL('DROP TABLE pmnENROLLMENT'); +PMN_DROPSQL('DROP TABLE enrollment'); END; / -CREATE TABLE pmnENROLLMENT ( +CREATE TABLE enrollment ( PATID varchar(50) NOT NULL, ENR_START_DATE date NOT NULL, ENR_END_DATE date NULL, @@ -238,11 +238,11 @@ CREATE TABLE pmnENROLLMENT ( BEGIN -PMN_DROPSQL('DROP TABLE pmnVITAL'); +PMN_DROPSQL('DROP TABLE vital'); END; / -CREATE TABLE pmnVITAL ( +CREATE TABLE vital ( VITALIID NUMBER(19) primary key, PATID varchar(50) NULL, ENCOUNTERID varchar(50) NULL, @@ -271,28 +271,28 @@ CREATE TABLE pmnVITAL ( / BEGIN -PMN_DROPSQL('DROP SEQUENCE pmnVITAL_seq'); +PMN_DROPSQL('DROP SEQUENCE vital_seq'); END; / -create sequence pmnVITAL_seq +create sequence vital_seq / -create or replace trigger pmnVITAL_trg -before insert on pmnVITAL +create or replace trigger vital_trg +before insert on vital for each row begin - select pmnVITAL_seq.nextval into :new.VITALIID from dual; + select vital_seq.nextval into :new.VITALIID from dual; end; / BEGIN -PMN_DROPSQL('DROP TABLE pmnPROCEDURE'); +PMN_DROPSQL('DROP TABLE procedures'); END; / -CREATE TABLE pmnPROCEDURE( +CREATE TABLE procedures( PROCEDURESID NUMBER(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NOT NULL, @@ -309,27 +309,27 @@ CREATE TABLE pmnPROCEDURE( / BEGIN -PMN_DROPSQL('DROP sequence pmnPROCEDURE_seq'); +PMN_DROPSQL('DROP sequence procedures_seq'); END; / -create sequence pmnPROCEDURE_seq +create sequence procedures_seq / -create or replace trigger pmnPROCEDURE_trg -before insert on pmnPROCEDURE +create or replace trigger procedures_trg +before insert on procedures for each row begin - select pmnPROCEDURE_seq.nextval into :new.PROCEDURESID from dual; + select procedures_seq.nextval into :new.PROCEDURESID from dual; end; / BEGIN -PMN_DROPSQL('DROP TABLE pmndiagnosis'); +PMN_DROPSQL('DROP TABLE diagnosis'); END; / -CREATE TABLE pmndiagnosis( +CREATE TABLE diagnosis( DIAGNOSISID NUMBER(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NOT NULL, @@ -349,17 +349,17 @@ CREATE TABLE pmndiagnosis( / BEGIN -PMN_DROPSQL('DROP sequence pmndiagnosis_seq'); +PMN_DROPSQL('DROP sequence diagnosis_seq'); END; / -create sequence pmndiagnosis_seq +create sequence diagnosis_seq / -create or replace trigger pmndiagnosis_trg -before insert on pmndiagnosis +create or replace trigger diagnosis_trg +before insert on diagnosis for each row begin - select pmndiagnosis_seq.nextval into :new.DIAGNOSISID from dual; + select diagnosis_seq.nextval into :new.DIAGNOSISID from dual; end; / @@ -367,11 +367,11 @@ end; BEGIN -PMN_DROPSQL('DROP TABLE pmnlabresults_cm'); +PMN_DROPSQL('DROP TABLE lab_result_cm'); END; / -CREATE TABLE pmnlabresults_cm( +CREATE TABLE lab_result_cm( LAB_RESULT_CM_ID NUMBER(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NULL, @@ -408,18 +408,18 @@ CREATE TABLE pmnlabresults_cm( BEGIN -PMN_DROPSQL('DROP SEQUENCE pmnlabresults_cm_seq'); +PMN_DROPSQL('DROP SEQUENCE lab_result_cm_seq'); END; / -create sequence pmnlabresults_cm_seq +create sequence lab_result_cm_seq / -create or replace trigger pmnlabresults_cm_trg -before insert on pmnlabresults_cm +create or replace trigger lab_result_cm_trg +before insert on lab_result_cm for each row begin - select pmnlabresults_cm_seq.nextval into :new.LAB_RESULT_CM_ID from dual; + select lab_result_cm_seq.nextval into :new.LAB_RESULT_CM_ID from dual; end; / @@ -485,10 +485,10 @@ INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANG / BEGIN -PMN_DROPSQL('DROP TABLE pmndeath'); +PMN_DROPSQL('DROP TABLE death'); END; / -CREATE TABLE pmndeath( +CREATE TABLE death( PATID varchar(50) NOT NULL, DEATH_DATE date NOT NULL, DEATH_DATE_IMPUTE varchar(2) NULL, @@ -498,10 +498,10 @@ CREATE TABLE pmndeath( / BEGIN -PMN_DROPSQL('DROP TABLE pmndeath_cause'); +PMN_DROPSQL('DROP TABLE death_cause'); END; / -CREATE TABLE pmndeath_cause( +CREATE TABLE death_cause( PATID varchar(50) NOT NULL, DEATH_CAUSE varchar(8) NOT NULL, DEATH_CAUSE_CODE varchar(2) NOT NULL, @@ -512,10 +512,10 @@ CREATE TABLE pmndeath_cause( / BEGIN -PMN_DROPSQL('DROP TABLE pmndispensing'); +PMN_DROPSQL('DROP TABLE dispensing'); END; / -CREATE TABLE pmndispensing( +CREATE TABLE dispensing( DISPENSINGID NUMBER(19) primary key, PATID varchar(50) NOT NULL, PRESCRIBINGID NUMBER(19) NULL, -- jgk fix 9/24 @@ -529,17 +529,17 @@ CREATE TABLE pmndispensing( BEGIN -PMN_DROPSQL('DROP sequence pmndispensing_seq'); +PMN_DROPSQL('DROP sequence dispensing_seq'); END; / -create sequence pmndispensing_seq +create sequence dispensing_seq / -create or replace trigger pmndispensing_trg -before insert on pmndispensing +create or replace trigger dispensing_trg +before insert on dispensing for each row begin - select pmndispensing_seq.nextval into :new.DISPENSINGID from dual; + select dispensing_seq.nextval into :new.DISPENSINGID from dual; end; / @@ -553,10 +553,10 @@ end; BEGIN -PMN_DROPSQL('DROP TABLE pmnprescribing'); +PMN_DROPSQL('DROP TABLE prescribing'); END; / -CREATE TABLE pmnprescribing( +CREATE TABLE prescribing( PRESCRIBINGID NUMBER(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NULL, @@ -579,25 +579,25 @@ CREATE TABLE pmnprescribing( BEGIN -PMN_DROPSQL('DROP sequence pmnprescribing_seq'); +PMN_DROPSQL('DROP sequence prescribing_seq'); END; / -create sequence pmnprescribing_seq +create sequence prescribing_seq / -create or replace trigger pmnprescribing_trg -before insert on pmnprescribing +create or replace trigger prescribing_trg +before insert on prescribing for each row begin - select pmnprescribing_seq.nextval into :new.PRESCRIBINGID from dual; + select prescribing_seq.nextval into :new.PRESCRIBINGID from dual; end; / BEGIN -PMN_DROPSQL('DROP TABLE pmnpcornet_trial'); +PMN_DROPSQL('DROP TABLE pcornet_trial'); END; / -CREATE TABLE pmnpcornet_trial( +CREATE TABLE pcornet_trial( PATID varchar(50) NOT NULL, TRIAL_ID varchar(20) NOT NULL, PARTICIPANTID varchar(50) NOT NULL, @@ -610,10 +610,10 @@ CREATE TABLE pmnpcornet_trial( / BEGIN -PMN_DROPSQL('DROP TABLE pmncondition'); +PMN_DROPSQL('DROP TABLE condition'); END; / -CREATE TABLE pmncondition( +CREATE TABLE condition( CONDITIONID NUMBER(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NULL, @@ -632,27 +632,27 @@ CREATE TABLE pmncondition( / BEGIN -PMN_DROPSQL('DROP sequence pmncondition_seq'); +PMN_DROPSQL('DROP sequence condition_seq'); END; / -create sequence pmncondition_seq +create sequence condition_seq / -create or replace trigger pmncondition_trg -before insert on pmncondition +create or replace trigger condition_trg +before insert on condition for each row begin - select pmncondition_seq.nextval into :new.CONDITIONID from dual; + select condition_seq.nextval into :new.CONDITIONID from dual; end; / BEGIN -PMN_DROPSQL('DROP TABLE pmnpro_cm'); +PMN_DROPSQL('DROP TABLE pro_cm'); END; / -CREATE TABLE pmnpro_cm( +CREATE TABLE pro_cm( PRO_CM_ID NUMBER(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NULL, @@ -670,25 +670,25 @@ CREATE TABLE pmnpro_cm( / BEGIN -PMN_DROPSQL('DROP sequence pmnpro_cm_seq'); +PMN_DROPSQL('DROP sequence pro_cm_seq'); END; / -create sequence pmnpro_cm_seq +create sequence pro_cm_seq / -create or replace trigger pmnpro_cm_trg -before insert on pmnpro_cm +create or replace trigger pro_cm_trg +before insert on pro_cm for each row begin - select pmnpro_cm_seq.nextval into :new.PRO_CM_ID from dual; + select pro_cm_seq.nextval into :new.PRO_CM_ID from dual; end; / BEGIN -PMN_DROPSQL('DROP TABLE pmnharvest'); +PMN_DROPSQL('DROP TABLE harvest'); END; / -CREATE TABLE pmnharvest( +CREATE TABLE harvest( NETWORKID varchar(10) NOT NULL, NETWORK_NAME varchar(20) NULL, DATAMARTID varchar(10) NOT NULL, @@ -735,10 +735,10 @@ CREATE TABLE pmnharvest( BEGIN -PMN_DROPSQL('DROP TABLE pmnENCOUNTER'); +PMN_DROPSQL('DROP TABLE encounter'); END; / -CREATE TABLE pmnENCOUNTER( +CREATE TABLE encounter( PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NOT NULL, ADMIT_DATE date NULL, @@ -764,10 +764,10 @@ CREATE TABLE pmnENCOUNTER( / BEGIN -PMN_DROPSQL('DROP TABLE pmndemographic'); +PMN_DROPSQL('DROP TABLE demographic'); END; / -CREATE TABLE pmndemographic( +CREATE TABLE demographic( PATID varchar(50) NOT NULL, BIRTH_DATE date NULL, BIRTH_TIME varchar(5) NULL, @@ -821,7 +821,7 @@ create or replace procedure PCORNetDemographic as sqltext varchar2(4000); cursor getsql is --1 -- S,R,NH - select 'insert into PMNDEMOGRAPHIC(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| ' select ''1'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| @@ -838,7 +838,7 @@ cursor getsql is and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' and sex.c_visualattributes like 'L%' union -- A - S,R,H -select 'insert into PMNDEMOGRAPHIC(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| +select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| ' select ''A'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| @@ -858,7 +858,7 @@ select 'insert into PMNDEMOGRAPHIC(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HI and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' and sex.c_visualattributes like 'L%' union --2 S, nR, nH - select 'insert into PMNDEMOGRAPHIC(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| ' select ''2'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| @@ -873,7 +873,7 @@ union --2 S, nR, nH where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' and sex.c_visualattributes like 'L%' union --3 -- nS,R, NH - select 'insert into PMNDEMOGRAPHIC(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| ' select ''3'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| @@ -888,7 +888,7 @@ union --3 -- nS,R, NH where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' and race.c_visualattributes like 'L%' union --B -- nS,R, H - select 'insert into PMNDEMOGRAPHIC(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| ' select ''B'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| @@ -906,7 +906,7 @@ union --B -- nS,R, H and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC\Y%' and hisp.c_visualattributes like 'L%' union --4 -- S, NR, H - select 'insert into PMNDEMOGRAPHIC(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| ' select ''4'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| @@ -921,7 +921,7 @@ union --4 -- S, NR, H where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' and sex.c_visualattributes like 'L%' union --5 -- NS, NR, H - select 'insert into PMNDEMOGRAPHIC(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| ' select ''5'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| @@ -934,7 +934,7 @@ union --5 -- NS, NR, H ' and lower(nvl(p.race_cd,''xx'')) in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'')' from dual union --6 -- NS, NR, nH - select 'insert into PMNDEMOGRAPHIC(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| ' select ''6'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| @@ -999,7 +999,7 @@ create or replace procedure PCORNetEncounter as sqltext varchar2(4000); begin -insert into pmnencounter(PATID,ENCOUNTERID,admit_date ,ADMIT_TIME , +insert into encounter(PATID,ENCOUNTERID,admit_date ,ADMIT_TIME , DISCHARGE_DATE ,DISCHARGE_TIME ,PROVIDERID ,FACILITY_LOCATION ,ENC_TYPE ,FACILITYID ,DISCHARGE_DISPOSITION , DISCHARGE_STATUS ,DRG ,DRG_TYPE ,ADMITTING_SOURCE) @@ -1015,13 +1015,13 @@ select distinct v.patient_num, v.encounter_num, 'NI' discharge_disposition /*CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END*/ , 'NI' discharge_status /* See TODO above CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END*/ , drg.drg, drg_type, 'NI' admitting_source /* see TODO above CASE WHEN _source IS NULL THEN 'NI' ELSE admitting_source END*/ -from i2b2visit v inner join pmndemographic d on v.patient_num=d.patid +from i2b2visit v inner join demographic d on v.patient_num=d.patid left outer join (select * from (select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from (select patient_num,encounter_num,drg_type,max(drg) drg from (select distinct f.patient_num,encounter_num,SUBSTR(c_fullname,22,2) drg_type,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,3) drg from i2b2fact f - inner join pmndemographic d on f.patient_num=d.patid + inner join demographic d on f.patient_num=d.patid inner join pcornet_enc enc on enc.c_basecode = f.concept_cd and enc.c_fullname like '\PCORI\ENCOUNTER\DRG\%') drg1 group by patient_num,encounter_num,drg_type) drg) drg where rn=1) drg -- This section is bugfixed to only include 1 drg if multiple DRG types exist in a single encounter... @@ -1047,7 +1047,7 @@ PMN_DROPSQL('DROP TABLE sourcefact'); sqltext := 'create table sourcefact as '|| 'select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname '|| 'from i2b2fact factline '|| - 'inner join pmnENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| + 'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| 'inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode '|| 'where dxsource.c_fullname like ''\PCORI_MOD\CONDITION_OR_DX\%'''; PMN_EXECUATESQL(sqltext); @@ -1058,19 +1058,19 @@ PMN_DROPSQL('DROP TABLE pdxfact'); sqltext := 'create table pdxfact as '|| 'select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode pdxsource,dxsource.c_fullname '|| 'from i2b2fact factline '|| - 'inner join pmnENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| + 'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| 'inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode '|| 'and dxsource.c_fullname like ''\PCORI_MOD\PDX\%'''; PMN_EXECUATESQL(sqltext); -sqltext := 'insert into pmndiagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, pdx) '|| +sqltext := 'insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, pdx) '|| 'select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, factline.start_date, factline.provider_id, diag.pcori_basecode, '|| 'SUBSTR(diag.c_fullname,18,2) dxtype, '|| ' CASE WHEN enc_type=''AV'' THEN ''FI'' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,'':'')+1,2) ,''NI'')END, '|| ' nvl(SUBSTR(pdxsource,INSTR(pdxsource, '':'')+1,2),''NI'') '|| -- jgk bugfix 9/28/15 'from i2b2fact factline '|| -'inner join pmnENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| +'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| ' left outer join sourcefact '|| 'on factline.patient_num=sourcefact.patient_num '|| 'and factline.encounter_num=sourcefact.encounter_num '|| @@ -1107,18 +1107,18 @@ PMN_DROPSQL('DROP TABLE sourcefact2'); sqltext := 'create table sourcefact2 as '|| 'select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname '|| 'from i2b2fact factline '|| - 'inner join pmnENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| + 'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| 'inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode '|| 'where dxsource.c_fullname like ''\PCORI_MOD\CONDITION_OR_DX\%'''; PMN_EXECUATESQL(sqltext); -sqltext := 'insert into pmncondition (patid, encounterid, report_date, resolve_date, condition, condition_type, condition_status, condition_source) '|| +sqltext := 'insert into condition (patid, encounterid, report_date, resolve_date, condition, condition_type, condition_status, condition_source) '|| 'select distinct factline.patient_num, min(factline.encounter_num) encounterid, min(factline.start_date) report_date, NVL(max(factline.end_date),null) resolve_date, diag.pcori_basecode, '|| 'SUBSTR(diag.c_fullname,18,2) condition_type, '|| ' NVL2(max(factline.end_date) , ''RS'', ''NI'') condition_status, '|| -- Imputed so might not be entirely accurate ' NVL(SUBSTR(max(dxsource),INSTR(max(dxsource), '':'')+1,2),''NI'') condition_source '|| 'from i2b2fact factline '|| -'inner join pmnENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| +'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| 'inner join pcornet_diag diag on diag.c_basecode = factline.concept_cd '|| ' left outer join sourcefact2 sf '|| 'on factline.patient_num=sf.patient_num '|| @@ -1143,12 +1143,12 @@ end PCORNetCondition; create or replace procedure PCORNetProcedure as begin -insert into pmnprocedure( +insert into procedures( patid, encounterid, enc_type, admit_date, providerid, px, px_type) select distinct fact.patient_num, enc.encounterid, enc.enc_type, fact.start_date, fact.provider_id, SUBSTR(pr.pcori_basecode,INSTR(pr.pcori_basecode, ':')+1,11) px, SUBSTR(pr.c_fullname,18,2) pxtype from i2b2fact fact - inner join pmnENCOUNTER enc on enc.patid = fact.patient_num and enc.encounterid = fact.encounter_Num + inner join encounter enc on enc.patid = fact.patient_num and enc.encounterid = fact.encounter_Num inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd where pr.c_fullname like '\PCORI\PROCEDURE\%'; @@ -1164,7 +1164,7 @@ end PCORNetProcedure; create or replace procedure PCORNetVital as begin -- jgk: I took out admit_date - it doesn't appear in the scheme. Now in SQLServer format - date, substring, name on inner select, no nested with. Added modifiers and now use only pathnames, not codes. -insert into pmnVITAL(patid, encounterid, measure_date, measure_time,vital_source,ht, wt, diastolic, systolic, original_bmi, bp_position,smoking,tobacco,tobacco_type) +insert into vital(patid, encounterid, measure_date, measure_time,vital_source,ht, wt, diastolic, systolic, original_bmi, bp_position,smoking,tobacco,tobacco_type) select patid, encounterid, to_date(measure_date,'rrrr-mm-dd') measure_date, measure_time,vital_source,ht, wt, diastolic, systolic, original_bmi, bp_position,smoking,tobacco, case when tobacco in ('02','03','04') then -- no tobacco case when smoking in ('03','04') then '04' -- no smoking @@ -1194,7 +1194,7 @@ from ( , case when vit.pcori_code like '\PCORI\VITAL\TOBACCO\SMOKING\%' then vit.pcori_basecode else null end smoking , case when vit.pcori_code like '\PCORI\VITAL\TOBACCO\__\%' then vit.pcori_basecode else null end unk_tobacco , enc.admit_date - from pmndemographic pd + from demographic pd left join ( select obs.patient_num patid, obs.encounter_num encounterid, @@ -1223,7 +1223,7 @@ from ( where pm.c_fullname like bp.concept_path || '%' ) codes on codes.concept_cd = obs.concept_cd ) vit on vit.patid = pd.patid - join pmnencounter enc on enc.encounterid = vit.encounterid + join encounter enc on enc.encounterid = vit.encounterid ) x where ht is not null or wt is not null @@ -1247,11 +1247,11 @@ end PCORNetVital; create or replace procedure PCORNetEnroll as begin -INSERT INTO pmnENROLLMENT(PATID, ENR_START_DATE, ENR_END_DATE, CHART, BASIS) +INSERT INTO enrollment(PATID, ENR_START_DATE, ENR_END_DATE, CHART, BASIS) select x.patient_num patid, case when l.patient_num is not null then l.period_start else enr_start end enr_start_date , case when l.patient_num is not null then l.period_end when enr_end_end>enr_end then enr_end_end else enr_end end enr_end_date , 'Y' chart, case when l.patient_num is not null then 'A' else 'E' end basis from - (select patient_num, min(start_date) enr_start,max(start_date) enr_end,max(end_date) enr_end_end from i2b2visit where patient_num in (select patid from pmndemographic) group by patient_num) x + (select patient_num, min(start_date) enr_start,max(start_date) enr_end,max(end_date) enr_end_end from i2b2visit where patient_num in (select patid from demographic) group by patient_num) x left outer join i2b2loyalty_patients l on l.patient_num=x.patient_num; end PCORNetEnroll; @@ -1275,7 +1275,7 @@ drop table location; create table priority as ( select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode PRIORITY from i2b2fact - inner join pmnENCOUNTER enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num + inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode where c_fullname LIKE '\PCORI_MOD\PRIORITY\%' ); @@ -1283,7 +1283,7 @@ create table priority as ( create table location as ( select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode RESULT_LOC from i2b2fact - inner join pmnENCOUNTER enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num + inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode where c_fullname LIKE '\PCORI_MOD\RESULT_LOC\%' ); @@ -1301,7 +1301,7 @@ PMN_DROPSQL('DROP TABLE priority'); sqltext := 'create table priority as '|| '(select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode PRIORITY '|| 'from i2b2fact '|| -'inner join pmnENCOUNTER enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num '|| +'inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num '|| 'inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode '|| 'where c_fullname LIKE ''\PCORI_MOD\PRIORITY\%'') '; @@ -1312,14 +1312,14 @@ PMN_DROPSQL('DROP TABLE location'); sqltext := 'create table location as '|| '(select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode RESULT_LOC '|| 'from i2b2fact '|| -'inner join pmnENCOUNTER enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num '|| +'inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num '|| 'inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode '|| 'where c_fullname LIKE ''\PCORI_MOD\RESULT_LOC\%'') '; PMN_EXECUATESQL(sqltext); -INSERT INTO pmnlabresults_cm +INSERT INTO lab_result_cm (PATID ,ENCOUNTERID ,LAB_NAME @@ -1383,7 +1383,7 @@ NULL RAW_ORDER_DEPT, NULL RAW_FACILITY_CODE FROM i2b2fact M -inner join pmnENCOUNTER enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num -- Constraint to selected encounters +inner join encounter enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num -- Constraint to selected encounters inner join pcornet_lab lab on lab.c_basecode = M.concept_cd and lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' inner JOIN pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname @@ -1418,7 +1418,7 @@ END PCORNetLabResultCM; create or replace procedure PCORNetHarvest as begin -INSERT INTO pmnharvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMENT_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) +INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMENT_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) VALUES('&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, 01, 02, 1,1,2,1,2,1,2,1,2,1,1,2,2,1,1,1,2,1,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,null,current_date,null,null,null); end PCORNetHarvest; @@ -1443,7 +1443,7 @@ drop table supply; create table basis as ( select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2fact basis - inner join pmnENCOUNTER enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num + inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num join pcornet_med basiscode on basis.modifier_cd = basiscode.c_basecode and basiscode.c_fullname like '\PCORI_MOD\RX_BASIS\%' @@ -1451,7 +1451,7 @@ create table basis as ( create table freq as ( select pcori_basecode,encounter_num,concept_cd from i2b2fact freq - inner join pmnENCOUNTER enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num + inner join encounter enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num join pcornet_med freqcode on freq.modifier_cd = freqcode.c_basecode and freqcode.c_fullname like '\PCORI_MOD\RX_FREQUENCY\%' @@ -1459,7 +1459,7 @@ create table freq as ( create table quantity as ( select nval_num,encounter_num,concept_cd from i2b2fact quantity - inner join pmnENCOUNTER enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num + inner join encounter enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num join pcornet_med quantitycode on quantity.modifier_cd = quantitycode.c_basecode and quantitycode.c_fullname like '\PCORI_MOD\RX_QUANTITY\' @@ -1467,7 +1467,7 @@ create table quantity as ( create table refills as ( select nval_num,encounter_num,concept_cd from i2b2fact refills - inner join pmnENCOUNTER enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num + inner join encounter enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num join pcornet_med refillscode on refills.modifier_cd = refillscode.c_basecode and refillscode.c_fullname like '\PCORI_MOD\RX_REFILLS\' @@ -1475,7 +1475,7 @@ create table refills as ( create table supply as ( select nval_num,encounter_num,concept_cd from i2b2fact supply - inner join pmnENCOUNTER enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num + inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num join pcornet_med supplycode on supply.modifier_cd = supplycode.c_basecode and supplycode.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\' @@ -1493,7 +1493,7 @@ begin PMN_DROPSQL('DROP TABLE basis'); sqltext := 'create table basis as '|| '(select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2fact basis '|| -' inner join pmnENCOUNTER enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num '|| +' inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num '|| ' join pcornet_med basiscode '|| ' on basis.modifier_cd = basiscode.c_basecode '|| ' and basiscode.c_fullname like ''\PCORI_MOD\RX_BASIS\%'') '; @@ -1502,7 +1502,7 @@ PMN_EXECUATESQL(sqltext); PMN_DROPSQL('DROP TABLE freq'); sqltext := 'create table freq as '|| '(select pcori_basecode,encounter_num,concept_cd from i2b2fact freq '|| -' inner join pmnENCOUNTER enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num '|| +' inner join encounter enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num '|| ' join pcornet_med freqcode '|| ' on freq.modifier_cd = freqcode.c_basecode '|| ' and freqcode.c_fullname like ''\PCORI_MOD\RX_FREQUENCY\%'') '; @@ -1511,7 +1511,7 @@ PMN_EXECUATESQL(sqltext); PMN_DROPSQL('DROP TABLE quantity'); sqltext := 'create table quantity as '|| '(select nval_num,encounter_num,concept_cd from i2b2fact quantity '|| -' inner join pmnENCOUNTER enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num '|| +' inner join encounter enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num '|| ' join pcornet_med quantitycode '|| ' on quantity.modifier_cd = quantitycode.c_basecode '|| ' and quantitycode.c_fullname like ''\PCORI_MOD\RX_QUANTITY\'') '; @@ -1521,7 +1521,7 @@ PMN_EXECUATESQL(sqltext); PMN_DROPSQL('DROP TABLE refills'); sqltext := 'create table refills as '|| '(select nval_num,encounter_num,concept_cd from i2b2fact refills '|| -' inner join pmnENCOUNTER enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num '|| +' inner join encounter enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num '|| ' join pcornet_med refillscode '|| ' on refills.modifier_cd = refillscode.c_basecode '|| ' and refillscode.c_fullname like ''\PCORI_MOD\RX_REFILLS\'') '; @@ -1530,7 +1530,7 @@ PMN_EXECUATESQL(sqltext); PMN_DROPSQL('DROP TABLE supply'); sqltext := 'create table supply as '|| '(select nval_num,encounter_num,concept_cd from i2b2fact supply '|| -' inner join pmnENCOUNTER enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| +' inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| ' join pcornet_med supplycode '|| ' on supply.modifier_cd = supplycode.c_basecode '|| ' and supplycode.c_fullname like ''\PCORI_MOD\RX_DAYS_SUPPLY\'') '; @@ -1538,7 +1538,7 @@ PMN_EXECUATESQL(sqltext); -- insert data with outer joins to ensure all records are included even if some data elements are missing -insert into pmnprescribing ( +insert into prescribing ( PATID ,encounterid ,RX_PROVIDERID @@ -1559,7 +1559,7 @@ insert into pmnprescribing ( select distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH:MI'), m.start_date start_date, m.end_date, mo.pcori_cui ,quantity.nval_num quantity, refills.nval_num refills, supply.nval_num supply, freq.pcori_basecode frequency, basis.pcori_basecode basis from i2b2fact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode -inner join pmnENCOUNTER enc on enc.encounterid = m.encounter_Num +inner join encounter enc on enc.encounterid = m.encounter_Num -- TODO: This join adds several minutes to the load - must be debugged left join basis @@ -1617,7 +1617,7 @@ begin PMN_DROPSQL('DROP TABLE supply'); sqltext := 'create table supply as '|| '(select nval_num,encounter_num,concept_cd from i2b2fact supply '|| -' inner join pmnENCOUNTER enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| +' inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| ' join pcornet_med supplycode '|| ' on supply.modifier_cd = supplycode.c_basecode '|| ' and supplycode.c_fullname like ''\PCORI_MOD\RX_DAYS_SUPPLY\'' ) '; @@ -1634,7 +1634,7 @@ PMN_EXECUATESQL(sqltext); -- insert data with outer joins to ensure all records are included even if some data elements are missing -insert into pmndispensing ( +insert into dispensing ( PATID ,PRESCRIBINGID ,DISPENSE_DATE -- using start_date from i2b2 @@ -1647,16 +1647,16 @@ select m.patient_num, null,m.start_date, NVL(mo.pcori_ndc,'NA') ,max(supply.nval_num) sup, max(amount.nval_num) amt from i2b2fact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode -inner join pmnENCOUNTER enc on enc.encounterid = m.encounter_Num - +inner join encounter enc on enc.encounterid = m.encounter_Num + -- jgk bugfix 11/2 - we weren't filtering dispensing events inner join (select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2fact basis - inner join pmnENCOUNTER enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num + inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num join pcornet_med basiscode on basis.modifier_cd = basiscode.c_basecode and basiscode.c_fullname='\PCORI_MOD\RX_BASIS\DI\') basis on m.encounter_num = basis.encounter_num - and m.concept_cd = basis.concept_Cd + and m.concept_cd = basis.concept_Cd left join supply on m.encounter_num = supply.encounter_num @@ -1688,52 +1688,52 @@ i2b2procs number; i2b2lcs number; pmnpats number; -pmnencounters number; +encounters number; pmndx number; pmnprocs number; pmnfacts number; pmnenroll number; -pmnvital number; +vital number; pmnlabs number; -pmnprescribings number; -pmndispensings number; +prescribings number; +dispensings number; pmncond number; v_runid number; begin select count(*) into i2b2Pats from i2b2patient; -select count(*) into i2b2Encounters from i2b2visit i inner join pmndemographic d on i.patient_num=d.patid; +select count(*) into i2b2Encounters from i2b2visit i inner join demographic d on i.patient_num=d.patid; -select count(*) into pmnPats from pmndemographic; -select count(*) into pmnencounters from pmnencounter e ; -select count(*) into pmndx from pmndiagnosis; -select count(*) into pmnprocs from pmnprocedure; +select count(*) into pmnPats from demographic; +select count(*) into encounters from encounter e ; +select count(*) into pmndx from diagnosis; +select count(*) into pmnprocs from procedures; -select count(*) into pmncond from pmncondition; -select count(*) into pmnenroll from pmnenrollment; -select count(*) into pmnvital from pmnvital; -select count(*) into pmnlabs from pmnlabresults_cm; -select count(*) into pmnprescribings from pmnprescribing; -select count(*) into pmndispensings from pmndispensing; +select count(*) into pmncond from condition; +select count(*) into pmnenroll from enrollment; +select count(*) into vital from vital; +select count(*) into pmnlabs from lab_result_cm; +select count(*) into prescribings from prescribing; +select count(*) into dispensings from dispensing; select max(runid) into v_runid from i2pReport; v_runid := v_runid + 1; insert into i2pReport values( v_runid, SYSDATE, 'Pats', i2b2pats, pmnpats, i2b2pats-pmnpats); insert into i2pReport values( v_runid, SYSDATE, 'Enrollment', i2b2pats, pmnenroll, i2b2pats-pmnpats); -insert into i2pReport values(v_runid, SYSDATE, 'Encounters', i2b2Encounters, pmnEncounters, i2b2encounters-pmnencounters); +insert into i2pReport values(v_runid, SYSDATE, 'Encounters', i2b2Encounters, encounters, i2b2encounters-encounters); insert into i2pReport values(v_runid, SYSDATE, 'DX', null, pmndx, null); insert into i2pReport values(v_runid, SYSDATE, 'PX', null, pmnprocs, null); insert into i2pReport values(v_runid, SYSDATE, 'Condition', null, pmncond, null); -insert into i2pReport values(v_runid, SYSDATE, 'Vital', null, pmnvital, null); +insert into i2pReport values(v_runid, SYSDATE, 'Vital', null, vital, null); insert into i2pReport values(v_runid, SYSDATE, 'Labs', null, pmnlabs, null); -insert into i2pReport values(v_runid, SYSDATE, 'Prescribing', null, pmnprescribings,null); -insert into i2pReport values(v_runid, SYSDATE, 'Dispensing', null, pmndispensings, null); +insert into i2pReport values(v_runid, SYSDATE, 'Prescribing', null, prescribings,null); +insert into i2pReport values(v_runid, SYSDATE, 'Dispensing', null, dispensings, null); end pcornetReport; / From 3d412932a93cebfb2f8243301052106a43064a52 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 29 Feb 2016 15:10:22 -0600 Subject: [PATCH 011/507] ENR_BASIS was mistakenly named BASIS in ENROLLMENT table. --- Oracle/PCORNetLoader_ora.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index edb560f..6511de5 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1,7 +1,7 @@ ------------------------------------------------------------------------------------------- -- PCORNetLoader Script -- Orignal MSSQL Verion Contributors: Jeff Klann, PhD; Aaron Abend; Arturo Torres --- Translate to Oracle version: by Kun Wei(Wake Forest) +-- Translate to Oracle version: by Kun Wei(Wake Forest) -- Version 0.6.2, bugfix release, 1/6/16 (create table and pcornetreport bugs) -- Version 6.01, release to SCILHS, 10/15/15 -- Prescribing/dispensing bugfixes (untested) inserted by Jeff Klann 12/10/15 @@ -229,7 +229,7 @@ CREATE TABLE enrollment ( ENR_START_DATE date NOT NULL, ENR_END_DATE date NULL, CHART varchar(1) NULL, - BASIS varchar(1) NOT NULL, + ENR_BASIS varchar(1) NOT NULL, RAW_CHART varchar(50) NULL, RAW_BASIS varchar(50) NULL ) @@ -1247,7 +1247,7 @@ end PCORNetVital; create or replace procedure PCORNetEnroll as begin -INSERT INTO enrollment(PATID, ENR_START_DATE, ENR_END_DATE, CHART, BASIS) +INSERT INTO enrollment(PATID, ENR_START_DATE, ENR_END_DATE, CHART, ENR_BASIS) select x.patient_num patid, case when l.patient_num is not null then l.period_start else enr_start end enr_start_date , case when l.patient_num is not null then l.period_end when enr_end_end>enr_end then enr_end_end else enr_end end enr_end_date , 'Y' chart, case when l.patient_num is not null then 'A' else 'E' end basis from From 1389c38a54b00f89c9b59751ca056afc1918941b Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 29 Feb 2016 15:29:33 -0600 Subject: [PATCH 012/507] Several table ids and foreign keys were number fields not varchars. --- Oracle/PCORNetLoader_ora.sql | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 6511de5..e881ab3 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -243,7 +243,7 @@ END; / CREATE TABLE vital ( - VITALIID NUMBER(19) primary key, + VITALIID varchar(19) primary key, PATID varchar(50) NULL, ENCOUNTERID varchar(50) NULL, MEASURE_DATE date NULL, @@ -293,7 +293,7 @@ END; / CREATE TABLE procedures( - PROCEDURESID NUMBER(19) primary key, + PROCEDURESID varchar(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NOT NULL, ENC_TYPE varchar(2) NULL, @@ -330,7 +330,7 @@ END; / CREATE TABLE diagnosis( - DIAGNOSISID NUMBER(19) primary key, + DIAGNOSISID varchar(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NOT NULL, ENC_TYPE varchar(2) NULL, @@ -372,7 +372,7 @@ END; / CREATE TABLE lab_result_cm( - LAB_RESULT_CM_ID NUMBER(19) primary key, + LAB_RESULT_CM_ID varchar(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NULL, LAB_NAME varchar(10) NULL, @@ -516,9 +516,9 @@ PMN_DROPSQL('DROP TABLE dispensing'); END; / CREATE TABLE dispensing( - DISPENSINGID NUMBER(19) primary key, + DISPENSINGID varchar(19) primary key, PATID varchar(50) NOT NULL, - PRESCRIBINGID NUMBER(19) NULL, -- jgk fix 9/24 + PRESCRIBINGID varchar(19) NULL, DISPENSE_DATE date NOT NULL, NDC varchar (11) NOT NULL, DISPENSE_SUP int, @@ -557,7 +557,7 @@ PMN_DROPSQL('DROP TABLE prescribing'); END; / CREATE TABLE prescribing( - PRESCRIBINGID NUMBER(19) primary key, + PRESCRIBINGID varchar(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NULL, RX_PROVIDERID varchar(50) NULL, -- NOTE: The spec has a _ before the ID, but this is inconsistent. @@ -614,7 +614,7 @@ PMN_DROPSQL('DROP TABLE condition'); END; / CREATE TABLE condition( - CONDITIONID NUMBER(19) primary key, + CONDITIONID varchar(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NULL, REPORT_DATE date NULL, @@ -653,7 +653,7 @@ PMN_DROPSQL('DROP TABLE pro_cm'); END; / CREATE TABLE pro_cm( - PRO_CM_ID NUMBER(19) primary key, + PRO_CM_ID varchar(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NULL, PRO_ITEM varchar (7) NOT NULL, From d087c72e3b6f079c799ef2ed37fc2f3ec02548c7 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 29 Feb 2016 15:31:36 -0600 Subject: [PATCH 013/507] Changed RX_FREQUENCY to varchar(2) to meet CDMv3 specification. --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index e881ab3..6bbd43c 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -568,7 +568,7 @@ CREATE TABLE prescribing( RX_QUANTITY int NULL, RX_REFILLS int NULL, RX_DAYS_SUPPLY int NULL, - RX_FREQUENCY int NULL, + RX_FREQUENCY varchar(2) NULL, RX_BASIS varchar (2) NULL, RXNORM_CUI int NULL, RAW_RX_MED_NAME varchar (50) NULL, From 9d3a11e693cc81dbbf8430f9856b2bb8e84ac766 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 29 Feb 2016 15:32:54 -0600 Subject: [PATCH 014/507] TRIALID mistakenly labeled TRIAL_ID in PCORNET_TRIAL table. --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 6bbd43c..7152ade 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -599,7 +599,7 @@ END; / CREATE TABLE pcornet_trial( PATID varchar(50) NOT NULL, - TRIAL_ID varchar(20) NOT NULL, + TRIALID varchar(20) NOT NULL, PARTICIPANTID varchar(50) NOT NULL, TRIAL_SITEID varchar(50) NULL, TRIAL_ENROLL_DATE date NULL, From 328435793fb0382a3ca2468f36e8453d1a70657c Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 29 Feb 2016 15:35:56 -0600 Subject: [PATCH 015/507] SPECIMEN_DATE_MGMT mistakenly named SPECIMENT_DATE_MGMT in HARVEST table. --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 7152ade..f888aac 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -708,7 +708,7 @@ CREATE TABLE harvest( RX_END_DATE_MGMT varchar(2) NULL, DISPENSE_DATE_MGMT varchar(2) NULL, LAB_ORDER_DATE_MGMT varchar(2) NULL, - SPECIMENT_DATE_MGMT varchar(2) NULL, + SPECIMEN_DATE_MGMT varchar(2) NULL, RESULT_DATE_MGMT varchar(2) NULL, MEASURE_DATE_MGMT varchar(2) NULL, ONSET_DATE_MGMT varchar(2) NULL, From 23a9d783994719eaa22cf8474c9e52aebdc40fcc Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 29 Feb 2016 16:43:03 -0600 Subject: [PATCH 016/507] Replaced another reference to SPECIMENT_DATE_MGMT. --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index f888aac..8b05b0b 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1418,7 +1418,7 @@ END PCORNetLabResultCM; create or replace procedure PCORNetHarvest as begin -INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMENT_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) +INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) VALUES('&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, 01, 02, 1,1,2,1,2,1,2,1,2,1,1,2,2,1,1,1,2,1,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,null,current_date,null,null,null); end PCORNetHarvest; From 37bc658622cfb99231b8d25d338f35928c66c1ac Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 29 Feb 2016 17:56:38 -0600 Subject: [PATCH 017/507] Add local path mappings and supporting scripts. --- Oracle/pcornet_mapping.csv | 167 +++++++++++++++++++++++++++++++++ Oracle/pcornet_mapping.ctl | 8 ++ Oracle/pcornet_mapping_ddl.sql | 4 + 3 files changed, 179 insertions(+) create mode 100644 Oracle/pcornet_mapping.csv create mode 100644 Oracle/pcornet_mapping.ctl create mode 100644 Oracle/pcornet_mapping_ddl.sql diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv new file mode 100644 index 0000000..825ecac --- /dev/null +++ b/Oracle/pcornet_mapping.csv @@ -0,0 +1,167 @@ +pcori_path,local_path +\PCORI\DEMOGRAPHIC\BIOBANK_FLAG\Y\,\i2b2\Specimens\ +\PCORI\DEMOGRAPHIC\HISPANIC\N\,"\i2b2\Demographics\Ethnicity\Non Hispanic, Latino or Spanish Origin\" +\PCORI\DEMOGRAPHIC\HISPANIC\NI\,\i2b2\Demographics\Ethnicity\Not Recorded\ +\PCORI\DEMOGRAPHIC\HISPANIC\OT\,\i2b2\Demographics\Ethnicity\Other\ +\PCORI\DEMOGRAPHIC\HISPANIC\Y\,\i2b2\Demographics\Ethnicity\Hispanic\ +\PCORI\DEMOGRAPHIC\HISPANIC\Y\,\i2b2\Demographics\Ethnicity\Hispanic or Latino\ +\PCORI\DEMOGRAPHIC\HISPANIC\Y\,"\i2b2\Demographics\Ethnicity\Hispanic, Latino or Spanish Origin\" +\PCORI\ENCOUNTER\ENC_TYPE\AV\,\i2b2\Visit Details\Encounter Type\102\ +\PCORI\ENCOUNTER\ENC_TYPE\AV\,\i2b2\Visit Details\Encounter Type\126\ +\PCORI\ENCOUNTER\ENC_TYPE\ED\,\i2b2\Visit Details\Encounter Type\103\ +\PCORI\ENCOUNTER\ENC_TYPE\IP\,\i2b2\Visit Details\Encounter Type\101\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\110\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\128\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\122\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\127\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\107\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\104\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\130\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\125\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\124\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\114\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\123\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\109\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\115\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\129\ +\PCORI\DEMOGRAPHIC\SEX\F\,\i2b2\Demographics\Gender\Female\ +\PCORI\DEMOGRAPHIC\SEX\M\,\i2b2\Demographics\Gender\Male\ +\PCORI\DEMOGRAPHIC\SEX\NI\,\i2b2\Demographics\Gender\Unknown\Unknown-@\ +\PCORI\DEMOGRAPHIC\SEX\UN\,\i2b2\Demographics\Gender\Unknown\Unknown-U\ +\PCORI\DEMOGRAPHIC\RACE\01\,\i2b2\Demographics\Race\American Indian or Alaskan Native\ +\PCORI\DEMOGRAPHIC\RACE\02\,\i2b2\Demographics\Race\Asian\ +\PCORI\DEMOGRAPHIC\RACE\03\,\i2b2\Demographics\Race\Black or African American\ +\PCORI\DEMOGRAPHIC\RACE\06\,\i2b2\Demographics\Race\Two Races\ +\PCORI\DEMOGRAPHIC\RACE\04\,\i2b2\Demographics\Race\Native Hawaiian or Other Pacific Islander\ +\PCORI\DEMOGRAPHIC\RACE\NI\,\i2b2\Demographics\Race\Not Recorded\ +\PCORI\DEMOGRAPHIC\RACE\OT\,\i2b2\Demographics\Race\Other\ +\PCORI\DEMOGRAPHIC\RACE\07\,\i2b2\Demographics\Race\Declined\ +\PCORI\DEMOGRAPHIC\RACE\05\,\i2b2\Demographics\Race\White or Caucasian\ +\PCORI\DIAGNOSIS\DX_TYPE\09\,\i2b2\Diagnoses\ +\PCORI\PROCEDURE\PX_TYPE\C4\,\i2b2\Procedures\PRC\Metathesaurus CPT Hierarchical Terms\ +\PCORI\PROCEDURE\PX_TYPE\09\,\i2b2\Procedures\PRC\ICD9 (Inpatient)\ +\PCORI\VITAL\HT\,\i2b2\Flowsheet\KU\GEN CARD\IP/OP HEIGHT/WEIGHT T:900445\KU GEN CARD G IP/OP PROCEDURE HEIGHT/WEI G:900612 I#1\HEIGHT M:11 I#1\ +\PCORI\VITAL\BP\DIASTOLIC\,\i2b2\Flowsheet\KU\GEN TMP CARD ECHO VITALS T:900426\KU GEN CARD GRP CV COMMON VITALS G:910000 I#2\BLOOD PRESSURE M:5 I#2\DIASTOLIC\ +\PCORI\VITAL\BP\SYSTOLIC\,\i2b2\Flowsheet\KU\GEN TMP CARD ECHO VITALS T:900426\KU GEN CARD GRP CV COMMON VITALS G:910000 I#2\BLOOD PRESSURE M:5 I#2\SYSTOLIC\ +\PCORI\VITAL\ORIGINAL_BMI\,\i2b2\Flowsheet\MODEL\VITAL SIGNS COMPLEX T:31010\KU IP GRP ADULT HEIGHT/WEIGHT G:300220 I#4\KU IP ROW BMI M:301070 I#6\ +\PCORI\VITAL\WT\,\i2b2\Flowsheet\KU\GEN CARD\IP/OP HEIGHT/WEIGHT T:900445\KU GEN CARD G IP/OP PROCEDURE HEIGHT/WEI G:900612 I#1\WEIGHT/SCALE M:14 I#2\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\253\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\09\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\10\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\67\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\256\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\12\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\276\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\275\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\294\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\293\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\283\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\285\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\284\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\286\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\287\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\271\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\272\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\266\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\268\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\269\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\257\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\369\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\66\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\11\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\251\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\314\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\307\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\309\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\308\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\325\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\327\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\326\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\316\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\318\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\317\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\319\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\321\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\320\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\304\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\306\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\305\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\298\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\300\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\299\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\301\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\303\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\302\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\NI\,\i2b2\Visit Details\Discharge Disposition Codes\0\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\260\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\200\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\100\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\E\,\i2b2\Visit Details\Discharge Disposition Codes\20\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\43\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\06\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\01\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\261\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\50\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\51\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\68\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\254\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\259\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\258\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\07\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\15\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\366\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\262\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\63\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\64\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\255\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\264\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\65\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\62\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\250\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\02\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\03\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\30\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\263\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\252\ +\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\365\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\IP\,\i2b2\Visit Details\Discharge Disposition Codes\253\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\SH\,\i2b2\Visit Details\Discharge Disposition Codes\09\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\67\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\256\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\257\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\369\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\66\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\11\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\251\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\NI\,\i2b2\Visit Details\Discharge Disposition Codes\0\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\260\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\200\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\100\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\43\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\HH\,\i2b2\Visit Details\Discharge Disposition Codes\06\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\HO\,\i2b2\Visit Details\Discharge Disposition Codes\01\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\HS\,\i2b2\Visit Details\Discharge Disposition Codes\261\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\HS\,\i2b2\Visit Details\Discharge Disposition Codes\50\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\HS\,\i2b2\Visit Details\Discharge Disposition Codes\51\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\RH\,\i2b2\Visit Details\Discharge Disposition Codes\68\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\254\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\259\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\RH\,\i2b2\Visit Details\Discharge Disposition Codes\258\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\AM\,\i2b2\Visit Details\Discharge Disposition Codes\07\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\AW\,\i2b2\Visit Details\Discharge Disposition Codes\15\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\AW\,\i2b2\Visit Details\Discharge Disposition Codes\366\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\IP\,\i2b2\Visit Details\Discharge Disposition Codes\262\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\IP\,\i2b2\Visit Details\Discharge Disposition Codes\63\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\NH\,\i2b2\Visit Details\Discharge Disposition Codes\64\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\255\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\RH\,\i2b2\Visit Details\Discharge Disposition Codes\264\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\OT\,\i2b2\Visit Details\Discharge Disposition Codes\65\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\RH\,\i2b2\Visit Details\Discharge Disposition Codes\62\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\HO\,\i2b2\Visit Details\Discharge Disposition Codes\250\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\IP\,\i2b2\Visit Details\Discharge Disposition Codes\02\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\NH\,\i2b2\Visit Details\Discharge Disposition Codes\03\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\SH\,\i2b2\Visit Details\Discharge Disposition Codes\30\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\IP\,\i2b2\Visit Details\Discharge Disposition Codes\263\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\HH\,\i2b2\Visit Details\Discharge Disposition Codes\252\ +\PCORI\ENCOUNTER\DISCHARGE_STATUS\HS\,\i2b2\Visit Details\Discharge Disposition Codes\365\ +\PCORI\ENCOUNTER\DRG_TYPE\02\,\i2b2\UHC\Diagnosis\Base MS DRG\ diff --git a/Oracle/pcornet_mapping.ctl b/Oracle/pcornet_mapping.ctl new file mode 100644 index 0000000..eaaa936 --- /dev/null +++ b/Oracle/pcornet_mapping.ctl @@ -0,0 +1,8 @@ +options (errors=0, skip=1) +load data +truncate into table pcornet_mapping +fields terminated by ',' optionally enclosed by '"' +trailing nullcols( + PCORI_PATH, + LOCAL_PATH + ) \ No newline at end of file diff --git a/Oracle/pcornet_mapping_ddl.sql b/Oracle/pcornet_mapping_ddl.sql new file mode 100644 index 0000000..12d6e87 --- /dev/null +++ b/Oracle/pcornet_mapping_ddl.sql @@ -0,0 +1,4 @@ +CREATE TABLE pcornet_mapping ( + "PCORI_PATH" VARCHAR2(2000), + "LOCAL_PATH" VARCHAR2(2000) + ); From 50653bf28e7d2e495fab59e5bf093d649b67d478 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 29 Feb 2016 17:58:24 -0600 Subject: [PATCH 018/507] Added SQL to make local terms leaves of PCORNet vital ontology. Thanks to Phillip Reeder at UTSW for example code. --- Oracle/pcornet_mapping.sql | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 Oracle/pcornet_mapping.sql diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql new file mode 100644 index 0000000..f919fec --- /dev/null +++ b/Oracle/pcornet_mapping.sql @@ -0,0 +1,43 @@ +/* Adapted from work by Phillip Reeder at UTSW + +Map local terms as leaves under the PCORNet terms. +*/ + +/* Delete any existing mapped terms +*/ +set echo on; +--select * +delete +from "&&i2b2_meta_schema".PCORNET_VITAL where sourcesystem_cd='MAPPING'; + +insert into "&&i2b2_meta_schema".PCORNET_VITAL +SELECT PCORNET_VITAL.C_HLEVEL+1, + PCORNET_VITAL.C_FULLNAME || i2b2.c_basecode || '\' as C_FULLNAME, + i2b2.c_basecode || ' ' || i2b2.c_name as C_NAME, + PCORNET_VITAL.C_SYNONYM_CD, + PCORNET_VITAL.C_VISUALATTRIBUTES, + PCORNET_VITAL.C_TOTALNUM, + i2b2.c_basecode as C_BASECODE, + PCORNET_VITAL.C_METADATAXML, + PCORNET_VITAL.C_FACTTABLECOLUMN, + PCORNET_VITAL.C_TABLENAME, + PCORNET_VITAL.C_COLUMNNAME, + PCORNET_VITAL.C_COLUMNDATATYPE, + PCORNET_VITAL.C_OPERATOR, + PCORNET_VITAL.C_FULLNAME || i2b2.c_basecode || '\' as C_DIMCODE, + PCORNET_VITAL.C_COMMENT, + PCORNET_VITAL.C_TOOLTIP, + PCORNET_VITAL.M_APPLIED_PATH, + PCORNET_VITAL.UPDATE_DATE, + PCORNET_VITAL.DOWNLOAD_DATE, + PCORNET_VITAL.IMPORT_DATE, + 'MAPPING' as SOURCESYSTEM_CD, + PCORNET_VITAL.VALUETYPE_CD, + PCORNET_VITAL.M_EXCLUSION_CD, + PCORNET_VITAL.C_PATH, + PCORNET_VITAL.C_SYMBOL, + PCORNET_VITAL.PCORI_BASECODE +FROM "&&i2b2_meta_schema".PCORNET_VITAL join pcornet_mapping on pcornet_mapping.PCORI_PATH=PCORNET_VITAL.c_fullname and pcornet_mapping.local_path is not null +join "&&i2b2_meta_schema"."&&terms_table" i2b2 on i2b2.c_fullname=pcornet_mapping.local_path; + +commit; From 131a7fee5c5738d99ecd9e50b3e81c005a378118 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 29 Feb 2016 17:58:56 -0600 Subject: [PATCH 019/507] Added helper script for running the local terminology mapping and SCILHS transform. --- Oracle/run-i2p-transform.sh | 76 +++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 Oracle/run-i2p-transform.sh diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh new file mode 100644 index 0000000..1a858fa --- /dev/null +++ b/Oracle/run-i2p-transform.sh @@ -0,0 +1,76 @@ +# Expected environment variables (put there by Jenkins, etc.) + +# Database SID +#export sid= +# User and password for CDM user +#export pcornet_cdm_user= +#export pcornet_cdm= + +#export i2b2_data_schema= +#export i2b2_meta_schema= +#export datamart_id= +#export datamart_name= +#export network_id= +#export network_name= + +# All i2b2 terms - used for local path mapping +#export terms_table= + +# Create/Load the local path mapping table +sqlplus /nolog < Date: Wed, 2 Mar 2016 10:05:47 -0600 Subject: [PATCH 020/507] Fixed typo in VITALID field label. --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 8b05b0b..b5624f6 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -243,7 +243,7 @@ END; / CREATE TABLE vital ( - VITALIID varchar(19) primary key, + VITALID varchar(19) primary key, PATID varchar(50) NULL, ENCOUNTERID varchar(50) NULL, MEASURE_DATE date NULL, From 2e61016c771c91795f8cc59420ec668ca3147f95 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 2 Mar 2016 10:47:21 -0600 Subject: [PATCH 021/507] Test to make sure we have about the same number of patients with diagnoses in i2b2 and CDM. --- Oracle/cdm_transform_tests.sql | 17 +++++++++++++++++ Oracle/run-i2p-transform.sh | 3 +++ 2 files changed, 20 insertions(+) create mode 100644 Oracle/cdm_transform_tests.sql diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql new file mode 100644 index 0000000..26c1bb5 --- /dev/null +++ b/Oracle/cdm_transform_tests.sql @@ -0,0 +1,17 @@ +/* Test to make sure we got about the same number of patients in the CDM +diagnoses that we do in i2b2. +*/ +with cdm as ( + select count(distinct patid) qty from diagnosis + ), +i2b2 as ( + select count(distinct patient_num) qty from i2b2fact + where concept_cd in ( + select concept_cd from "&&i2b2_data_schema".concept_dimension + where concept_path like '\i2b2\Diagnoses\ICD9\%' + ) + ), +diff as ( + select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct from cdm cross join i2b2 + ) +select case when diff.pct > 10 then 1/0 else 1 end diag_pat_count_ok from diff; diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 1a858fa..61035c6 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -72,5 +72,8 @@ start pcornet_mapping.sql -- SCILHS transform start PCORNetLoader_ora.sql +-- CDM transform tests +start cdm_transform_tests.sql + quit; EOF From ad96a9eef6c4a42dfd2693e659027fb29dfe6ab0 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 2 Mar 2016 10:50:25 -0600 Subject: [PATCH 022/507] Replace PCORNet ICD9 hierarchy with the local hierarchy that includes local diagnoses code leaves. - TODO: Consider using a replacement variable for the diagnoses terms rather than replacing what's in the PCORNet table. --- Oracle/pcornet_mapping.sql | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index f919fec..0df1f5e 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -41,3 +41,49 @@ FROM "&&i2b2_meta_schema".PCORNET_VITAL join pcornet_mapping on pcornet_mapping. join "&&i2b2_meta_schema"."&&terms_table" i2b2 on i2b2.c_fullname=pcornet_mapping.local_path; commit; + + +/* Replace PCORNet ICD9 diagnoses hierarchy with the local hierarchy filling in +the pcornet_basecode with the expected values. +*/ + +--select * +delete +from "&&i2b2_meta_schema".PCORNET_DIAG +where c_fullname like '\PCORI\DIAGNOSIS\09\%' +; + +select count(*), scheme from ( + select substr(c_basecode, 1, instr(c_basecode, ':')) scheme from "&&i2b2_meta_schema".PCORNET_DIAG + ) +group by scheme; + + +insert into "&&i2b2_meta_schema".PCORNET_DIAG +with terms_dxi as ( + select + cicd.code dxicd, ht.* + from + "&&i2b2_meta_schema"."&&terms_table" ht + -- TODO: Stop cheating by going back to Clarity + left join clarity.edg_current_icd9 cicd on to_char(cicd.dx_id) = replace(ht.c_basecode, 'KUH|DX_ID:', '') + where c_fullname like '\i2b2\Diagnoses\ICD9\%' order by c_hlevel + ) +select + td.c_hlevel, + replace(td.c_fullname, '\i2b2\Diagnoses\ICD9\', '\PCORI\DIAGNOSIS\09\') c_fullname, + td.c_name, td.c_synonym_cd, td.c_visualattributes, + td.c_totalnum, td.c_basecode, td.c_metadataxml, td.c_facttablecolumn, td.c_tablename, + td.c_columnname, td.c_columndatatype, td.c_operator, td.c_dimcode, td.c_comment, + td.c_tooltip, td.m_applied_path, td.update_date, td.download_date, td.import_date, + td.sourcesystem_cd, td.valuetype_cd, td.m_exclusion_cd, td.c_path, td.c_symbol, + case + when td.dxicd is not null then td.dxicd + when td.c_basecode like 'ICD9:%' then replace(td.c_basecode, 'ICD9:', '') + else null + end pcori_basecode +from terms_dxi td +order by c_hlevel +; + +commit; From f69a597f266877c707159bd4adec6a89da1a0f1c Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 2 Mar 2016 11:53:20 -0600 Subject: [PATCH 023/507] Gather table statistics on metadata tables which in some cases vastly improves performance of the transform. - Remove leftover debug code. --- Oracle/gather_table_stats.sql | 16 ++++++++++++++++ Oracle/pcornet_mapping.sql | 5 ----- Oracle/run-i2p-transform.sh | 3 +++ 3 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 Oracle/gather_table_stats.sql diff --git a/Oracle/gather_table_stats.sql b/Oracle/gather_table_stats.sql new file mode 100644 index 0000000..5db61c6 --- /dev/null +++ b/Oracle/gather_table_stats.sql @@ -0,0 +1,16 @@ +/* Metadata table statistics - experimentation shows vastly better performance +if this is done before the CDM transform. +*/ +begin + for rec in (select table_name + from dba_tables + where owner = '&&i2b2_meta_schema' and table_name like 'PCORNET%') + loop + DBMS_STATS.GATHER_TABLE_STATS ( + ownname => '"&&i2b2_meta_schema"', + tabname => rec.table_name, + estimate_percent => 40 + ); + end loop; +end; +/ diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 0df1f5e..4ea9a10 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -53,11 +53,6 @@ from "&&i2b2_meta_schema".PCORNET_DIAG where c_fullname like '\PCORI\DIAGNOSIS\09\%' ; -select count(*), scheme from ( - select substr(c_basecode, 1, instr(c_basecode, ':')) scheme from "&&i2b2_meta_schema".PCORNET_DIAG - ) -group by scheme; - insert into "&&i2b2_meta_schema".PCORNET_DIAG with terms_dxi as ( diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 61035c6..72089f5 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -69,6 +69,9 @@ define terms_table=${terms_table} -- Local terminology mapping start pcornet_mapping.sql +-- Prepare for transform +start gather_table_stats.sql + -- SCILHS transform start PCORNetLoader_ora.sql From 551d2fe782ac0d468329a7a6a5971c8bf5b50f01 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 2 Mar 2016 11:57:09 -0600 Subject: [PATCH 024/507] Oops, missed ID database link. --- Oracle/pcornet_mapping.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 4ea9a10..b99f004 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -61,7 +61,7 @@ with terms_dxi as ( from "&&i2b2_meta_schema"."&&terms_table" ht -- TODO: Stop cheating by going back to Clarity - left join clarity.edg_current_icd9 cicd on to_char(cicd.dx_id) = replace(ht.c_basecode, 'KUH|DX_ID:', '') + left join clarity.edg_current_icd9@id cicd on to_char(cicd.dx_id) = replace(ht.c_basecode, 'KUH|DX_ID:', '') where c_fullname like '\i2b2\Diagnoses\ICD9\%' order by c_hlevel ) select From 275c8968a7efbb9e7edb167ecccff1d87341618a Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 2 Mar 2016 12:00:56 -0600 Subject: [PATCH 025/507] Fixed one more instance of "vitaliid". --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index b5624f6..60d5a8d 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -282,7 +282,7 @@ create or replace trigger vital_trg before insert on vital for each row begin - select vital_seq.nextval into :new.VITALIID from dual; + select vital_seq.nextval into :new.VITALID from dual; end; / From e7434803d391b09ef2c79dd8ad5cfaeeaeeb236e Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 2 Mar 2016 12:20:22 -0600 Subject: [PATCH 026/507] Avoid selecting from dba_tables as it requires elevated privileges. --- Oracle/gather_table_stats.sql | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Oracle/gather_table_stats.sql b/Oracle/gather_table_stats.sql index 5db61c6..29947b4 100644 --- a/Oracle/gather_table_stats.sql +++ b/Oracle/gather_table_stats.sql @@ -1,14 +1,18 @@ /* Metadata table statistics - experimentation shows vastly better performance if this is done before the CDM transform. + +ACK: http://stackoverflow.com/questions/2242024/for-each-string-execute-a-function-procedure */ +declare +table_list sys.dbms_debug_vc2coll := sys.dbms_debug_vc2coll( + 'PCORNET_DEMO', 'PCORNET_DIAG', 'PCORNET_DIAG_BACKUP', 'PCORNET_ENC', + 'PCORNET_ENROLL', 'PCORNET_LAB', 'PCORNET_MED', 'PCORNET_PROC', 'PCORNET_VITAL'); begin - for rec in (select table_name - from dba_tables - where owner = '&&i2b2_meta_schema' and table_name like 'PCORNET%') + for t in table_list.first .. table_list.last loop DBMS_STATS.GATHER_TABLE_STATS ( ownname => '"&&i2b2_meta_schema"', - tabname => rec.table_name, + tabname => table_list(t), estimate_percent => 40 ); end loop; From bc652d230a89ae5919049c1dd139489ad2a6eb1f Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 2 Mar 2016 12:29:39 -0600 Subject: [PATCH 027/507] Oops, pcornet_diag_backup table was just for debugging. --- Oracle/gather_table_stats.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/gather_table_stats.sql b/Oracle/gather_table_stats.sql index 29947b4..94f1a5e 100644 --- a/Oracle/gather_table_stats.sql +++ b/Oracle/gather_table_stats.sql @@ -5,8 +5,8 @@ ACK: http://stackoverflow.com/questions/2242024/for-each-string-execute-a-functi */ declare table_list sys.dbms_debug_vc2coll := sys.dbms_debug_vc2coll( - 'PCORNET_DEMO', 'PCORNET_DIAG', 'PCORNET_DIAG_BACKUP', 'PCORNET_ENC', - 'PCORNET_ENROLL', 'PCORNET_LAB', 'PCORNET_MED', 'PCORNET_PROC', 'PCORNET_VITAL'); + 'PCORNET_DEMO', 'PCORNET_DIAG', 'PCORNET_ENC', 'PCORNET_ENROLL', 'PCORNET_LAB', + 'PCORNET_MED', 'PCORNET_PROC', 'PCORNET_VITAL'); begin for t in table_list.first .. table_list.last loop From 998e546389b71c4a2108e5eead77be7bc2995890 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 2 Mar 2016 16:50:52 -0600 Subject: [PATCH 028/507] Test to make sure most of the procedures in i2b2 made it in to the CDM. --- Oracle/cdm_transform_tests.sql | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 26c1bb5..7c5227a 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -15,3 +15,26 @@ diff as ( select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct from cdm cross join i2b2 ) select case when diff.pct > 10 then 1/0 else 1 end diag_pat_count_ok from diff; + + +/* Test to make sure we got about the same number of patients in the CDM +procedure that we do in i2b2. +*/ +with cdm as ( + select count(distinct patid) qty from procedures + ), +i2b2 as ( + select count(distinct patient_num) qty from ( + select * from i2b2fact + where concept_cd in ( + select concept_cd from "&&i2b2_data_schema".concept_dimension + where concept_path like '\i2b2\Procedures\%' + ) + ) + ), +diff as ( + select + ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct + from cdm cross join i2b2 + ) +select case when diff.pct > 10 then 1/0 else 1 end proc_pat_count_ok from diff; From cce0011b49873f2e6e2c481d6acd5e935f325441 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 2 Mar 2016 16:51:58 -0600 Subject: [PATCH 029/507] Add local hierarchies for procedures (ICD9/CPT). --- Oracle/pcornet_mapping.sql | 47 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index b99f004..cb2278a 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -82,3 +82,50 @@ order by c_hlevel ; commit; + +--select * +delete +from "&&i2b2_meta_schema".PCORNET_PROC +where c_fullname like '\PCORI\PROCEDURE\09\_%' +; + +/* Replace PCORNet Procedure (ICD9) hierarchy with the local hierarchy. +*/ +insert into "&&i2b2_meta_schema".PCORNET_PROC +select + ht.c_hlevel, + replace(ht.c_fullname, '\i2b2\Procedures\PRC\ICD9 (Inpatient)', '\PCORI\PROCEDURE\09') c_fullname, + ht.c_name, ht.c_synonym_cd, ht.c_visualattributes, + ht.c_totalnum, ht.c_basecode, ht.c_metadataxml, ht.c_facttablecolumn, ht.c_tablename, + ht.c_columnname, ht.c_columndatatype, ht.c_operator, ht.c_dimcode, ht.c_comment, + ht.c_tooltip, ht.m_applied_path, ht.update_date, ht.download_date, ht.import_date, + ht.sourcesystem_cd, ht.valuetype_cd, ht.m_exclusion_cd, ht.c_path, ht.c_symbol, + ht.c_basecode pcori_basecode +from + "&&i2b2_meta_schema"."&&terms_table" ht +where c_fullname like '\i2b2\Procedures\PRC\ICD9 (Inpatient)\_%' order by c_hlevel +; + + +--select * +delete +from "&&i2b2_meta_schema".PCORNET_PROC +where c_fullname like '\PCORI\PROCEDURE\C4\%' +; + +/* Replace PCORNet Procedure (ICD9) hierarchy with the local hierarchy. +*/ +insert into "&&i2b2_meta_schema".PCORNET_PROC +select + ht.c_hlevel, + replace(ht.c_fullname, '\i2b2\Procedures\PRC\Metathesaurus CPT Hierarchical Terms', '\PCORI\PROCEDURE\C4') c_fullname, + ht.c_name, ht.c_synonym_cd, ht.c_visualattributes, + ht.c_totalnum, ht.c_basecode, ht.c_metadataxml, ht.c_facttablecolumn, ht.c_tablename, + ht.c_columnname, ht.c_columndatatype, ht.c_operator, ht.c_dimcode, ht.c_comment, + ht.c_tooltip, ht.m_applied_path, ht.update_date, ht.download_date, ht.import_date, + ht.sourcesystem_cd, ht.valuetype_cd, ht.m_exclusion_cd, ht.c_path, ht.c_symbol, + ht.c_basecode pcori_basecode +from + "&&i2b2_meta_schema"."&&terms_table" ht +where c_fullname like '\i2b2\Procedures\PRC\Metathesaurus CPT Hierarchical Terms\%' order by c_hlevel +; From c7ded0609da8733e51c27e647c72e022507c2a72 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 3 Mar 2016 09:15:36 -0600 Subject: [PATCH 030/507] Added SAS script which produces data step views for use in PCORnet SAS queries. --- SAS/data_step_view_prep.sas | 317 ++++++++++++++++++++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 SAS/data_step_view_prep.sas diff --git a/SAS/data_step_view_prep.sas b/SAS/data_step_view_prep.sas new file mode 100644 index 0000000..bdcfe4f --- /dev/null +++ b/SAS/data_step_view_prep.sas @@ -0,0 +1,317 @@ +/******************************************************************************* +* Generate the data step views required by the PRCOnet Diagnostic Query +* (diagnostic.sas). +* +* Provided the following values below before execution: +* - ORC_SCHEMA +* - ORC_USERNAME +* - ORC_PASSWORD +* - OCR_HOSTNAME +* - ORC_SID +* - VIEW_LIB_PATH +* +*******************************************************************************/ + +ODS HTML CLOSE; +ODS HTML; + +libname oracdata oracle schema=ORC_SCEMA user=ORC_USERNAME pw=ORC_PASSWORD + path="(DESCRIPTION = + (ADDRESS = + (PROTOCOL = TCP) + (HOST = ORC_HOSTNAME) + (PORT = 1521) + ) + (CONNECT_DATA = + (SID = ORC_SID) + ) + )"; + +libname sasdata 'VIEW_LIB_PATH'; + + +***************************************************************; +* Create data step view for DEMOGRAPHIC +***************************************************************; + +data sasdata.DEMOGRAPHIC / view=sasdata.DEMOGRAPHIC; + set oracdata.DEMOGRAPHIC( + rename = ( + BIRTH_TIME = _BIRTH_TIME + ) + ) + ; + + BIRTH_TIME = input(_BIRTH_TIME, hhmmss.); + format BIRTH_TIME hhmm.; + drop _BIRTH_TIME; +run; + +/* +proc contents data=oracdata.DEMOGRAPHIC; +run; +proc print data=oracdata.DEMOGRAPHIC (firstobs=1 obs=10); +run; + +proc contents data=sasdata.DEMOGRAPHIC; +run; +proc print data=sasdata.DEMOGRAPHIC (firstobs=1 obs=10); +run; +*/ + + +***************************************************************; +* Create data step view for ENROLLMENT +***************************************************************; + +data sasdata.ENROLLMENT / view=sasdata.ENROLLMENT; + set oracdata.ENROLLMENT; +run; + +/* +proc contents data=oracdata.ENROLLMENT; +run; +proc print data=oracdata.ENROLLMENT (firstobs=1 obs=10); +run; + +proc contents data=sasdata.ENROLLMENT; +run; +proc print data=sasdata.ENROLLMENT (firstobs=1 obs=10); +run; +*/ + + +***************************************************************; +* Create data step view for ENCOUNTER +***************************************************************; + +data sasdata.ENCOUNTER / view=sasdata.ENCOUNTER; + set oracdata.ENCOUNTER( + rename = ( + ADMIT_TIME = _ADMIT_TIME + DISCHARGE_TIME = _DISCHARGE_TIME + ) + ) + ; + + ADMIT_TIME = input(_ADMIT_TIME, hhmmss.); + format ADMIT_TIME hhmm.; + drop _ADMIT_TIME; + + DISCHARGE_TIME = input(_DISCHARGE_TIME, hhmmss.); + format DISCHARGE_TIME hhmm.; + drop _DISCHARGE_TIME; +run; + +/* +proc contents data=oracdata.ENCOUNTER; +run; +proc print data=oracdata.ENCOUNTER (firstobs=1 obs=10); +run; + +proc contents data=sasdata.ENCOUNTER; +run; +proc print data=sasdata.ENCOUNTER (firstobs=1 obs=10); +run; +*/ + + +***************************************************************; +* Create data step view for DIAGNOSIS +***************************************************************; + +data sasdata.DIAGNOSIS / view=sasdata.DIAGNOSIS; + set oracdata.DIAGNOSIS; +run; + + +***************************************************************; +* Create data step view for PROCEDURES +***************************************************************; + +data sasdata.PROCEDURES / view=sasdata.PROCEDURES; + set oracdata.PROCEDURES; +run; + + +***************************************************************; +* Create data step view for VITAL +***************************************************************; + +data sasdata.VITAL / view=sasdata.VITAL; + set oracdata.VITAL( + rename = ( + MEASURE_TIME = _MEASURE_TIME + ) + ) + ; + + MEASURE_TIME = input(_MEASURE_TIME, hhmmss.); + format MEASURE_TIME hhmm.; + drop _MEASURE_TIME; +run; + +/* +proc contents data=oracdata.VITAL; +run; +proc print data=oracdata.VITAL (firstobs=1 obs=10); +run; + +proc contents data=sasdata.VITAL; +run; +proc print data=sasdata.VITAL (firstobs=1 obs=10); +run; +*/ + + +***************************************************************; +* Create data step view for DISPENSING +***************************************************************; + +data sasdata.DISPENSING / view=sasdata.DISPENSING; + set oracdata.DISPENSING; +run; + + +***************************************************************; +* Create data step view for LAB_RESULT_CM +***************************************************************; + +data sasdata.LAB_RESULT_CM / view=sasdata.LAB_RESULT_CM; + set oracdata.LAB_RESULT_CM( + rename = ( + RESULT_TIME = _RESULT_TIME + SPECIMEN_TIME = _SPECIMEN_TIME + /*RESULT_NUM = _RESULT_NUM*/ + ) + ) + ; + + RESULT_TIME = input(_RESULT_TIME, hhmmss.); + format RESULT_TIME hhmm.; + drop _RESULT_TIME; + + SPECIMEN_TIME = input(_SPECIMEN_TIME, hhmmss.); + format SPECIMEN_TIME hhmm.; + drop _SPECIMEN_TIME; + + /*RESULT_NUM = put(_RESULT_NUM, best8.); + drop _RESULT_NUM;*/ +run; + +/* +proc contents data=oracdata.LAB_RESULT_CM; +run; +proc print data=oracdata.LAB_RESULT_CM (firstobs=1 obs=10); +run; + +proc contents data=sasdata.LAB_RESULT_CM; +run; +proc print data=sasdata.LAB_RESULT_CM (firstobs=1 obs=10); +run; +*/ + + +***************************************************************; +* Create data step view for CONDITION +***************************************************************; + +data sasdata.CONDITION / view=sasdata.CONDITION; + set oracdata.CONDITION; +run; + + +***************************************************************; +* Create data step view for PRO_CM +***************************************************************; + +data sasdata.PRO_CM / view=sasdata.PRO_CM; + set oracdata.PRO_CM( + rename = ( + PRO_TIME = _PRO_TIME + ) + ) + ; + + PRO_TIME = input(_PRO_TIME, hhmmss.); + format PRO_TIME hhmm.; + drop _PRO_TIME; +run; + +/* +proc contents data=oracdata.PRO_CM; +run; +proc print data=oracdata.PRO_CM (firstobs=1 obs=10); +run; + +proc contents data=sasdata.PRO_CM; +run; +proc print data=sasdata.PRO_CM (firstobs=1 obs=10); +run; +*/ + + +***************************************************************; +* Create data step view for PRESCRIBING +***************************************************************; + +data sasdata.PRESCRIBING / view=sasdata.PRESCRIBING; + set oracdata.PRESCRIBING( + rename = ( + RX_ORDER_TIME = _RX_ORDER_TIME + ) + ) + ; + + RX_ORDER_TIME = input(_RX_ORDER_TIME, hhmmss.); + format RX_ORDER_TIME hhmm.; + drop _RX_ORDER_TIME; +run; + +/* +proc contents data=oracdata.PRESCRIBING; +run; +proc print data=oracdata.PRESCRIBING (firstobs=1 obs=10); +run; + +proc contents data=sasdata.PRESCRIBING; +run; +proc print data=sasdata.PRESCRIBING (firstobs=1 obs=10); +run; +*/ + + +***************************************************************; +* Create data step view for PCORNET_TRIAL +***************************************************************; + +data sasdata.PCORNET_TRIAL / view=sasdata.PCORNET_TRIAL; + set oracdata.PCORNET_TRIAL; +run; + + +***************************************************************; +* Create data step view for DEATH +***************************************************************; + +data sasdata.DEATH / view=sasdata.DEATH; + set oracdata.DEATH; +run; + + +***************************************************************; +* Create data step view for DEATH_CAUSE +***************************************************************; + +data sasdata.DEATH_CAUSE / view=sasdata.DEATH_CAUSE; + set oracdata.DEATH_CAUSE; +run; + + +***************************************************************; +* Create data step view for HARVEST +***************************************************************; + +data sasdata.HARVEST / view=sasdata.HARVEST; + set oracdata.HARVEST; +run; From 4e60d595069e4a636cdcb5386909159609978547 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 3 Mar 2016 09:49:23 -0600 Subject: [PATCH 031/507] Removed un-needed debug statements. --- SAS/data_step_view_prep.sas | 84 ------------------------------------- 1 file changed, 84 deletions(-) diff --git a/SAS/data_step_view_prep.sas b/SAS/data_step_view_prep.sas index bdcfe4f..98ac05f 100644 --- a/SAS/data_step_view_prep.sas +++ b/SAS/data_step_view_prep.sas @@ -47,18 +47,6 @@ data sasdata.DEMOGRAPHIC / view=sasdata.DEMOGRAPHIC; drop _BIRTH_TIME; run; -/* -proc contents data=oracdata.DEMOGRAPHIC; -run; -proc print data=oracdata.DEMOGRAPHIC (firstobs=1 obs=10); -run; - -proc contents data=sasdata.DEMOGRAPHIC; -run; -proc print data=sasdata.DEMOGRAPHIC (firstobs=1 obs=10); -run; -*/ - ***************************************************************; * Create data step view for ENROLLMENT @@ -68,18 +56,6 @@ data sasdata.ENROLLMENT / view=sasdata.ENROLLMENT; set oracdata.ENROLLMENT; run; -/* -proc contents data=oracdata.ENROLLMENT; -run; -proc print data=oracdata.ENROLLMENT (firstobs=1 obs=10); -run; - -proc contents data=sasdata.ENROLLMENT; -run; -proc print data=sasdata.ENROLLMENT (firstobs=1 obs=10); -run; -*/ - ***************************************************************; * Create data step view for ENCOUNTER @@ -103,18 +79,6 @@ data sasdata.ENCOUNTER / view=sasdata.ENCOUNTER; drop _DISCHARGE_TIME; run; -/* -proc contents data=oracdata.ENCOUNTER; -run; -proc print data=oracdata.ENCOUNTER (firstobs=1 obs=10); -run; - -proc contents data=sasdata.ENCOUNTER; -run; -proc print data=sasdata.ENCOUNTER (firstobs=1 obs=10); -run; -*/ - ***************************************************************; * Create data step view for DIAGNOSIS @@ -151,18 +115,6 @@ data sasdata.VITAL / view=sasdata.VITAL; drop _MEASURE_TIME; run; -/* -proc contents data=oracdata.VITAL; -run; -proc print data=oracdata.VITAL (firstobs=1 obs=10); -run; - -proc contents data=sasdata.VITAL; -run; -proc print data=sasdata.VITAL (firstobs=1 obs=10); -run; -*/ - ***************************************************************; * Create data step view for DISPENSING @@ -199,18 +151,6 @@ data sasdata.LAB_RESULT_CM / view=sasdata.LAB_RESULT_CM; drop _RESULT_NUM;*/ run; -/* -proc contents data=oracdata.LAB_RESULT_CM; -run; -proc print data=oracdata.LAB_RESULT_CM (firstobs=1 obs=10); -run; - -proc contents data=sasdata.LAB_RESULT_CM; -run; -proc print data=sasdata.LAB_RESULT_CM (firstobs=1 obs=10); -run; -*/ - ***************************************************************; * Create data step view for CONDITION @@ -238,18 +178,6 @@ data sasdata.PRO_CM / view=sasdata.PRO_CM; drop _PRO_TIME; run; -/* -proc contents data=oracdata.PRO_CM; -run; -proc print data=oracdata.PRO_CM (firstobs=1 obs=10); -run; - -proc contents data=sasdata.PRO_CM; -run; -proc print data=sasdata.PRO_CM (firstobs=1 obs=10); -run; -*/ - ***************************************************************; * Create data step view for PRESCRIBING @@ -268,18 +196,6 @@ data sasdata.PRESCRIBING / view=sasdata.PRESCRIBING; drop _RX_ORDER_TIME; run; -/* -proc contents data=oracdata.PRESCRIBING; -run; -proc print data=oracdata.PRESCRIBING (firstobs=1 obs=10); -run; - -proc contents data=sasdata.PRESCRIBING; -run; -proc print data=sasdata.PRESCRIBING (firstobs=1 obs=10); -run; -*/ - ***************************************************************; * Create data step view for PCORNET_TRIAL From 307edda9cb537f9566139b5c443c3c12681728cb Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Thu, 3 Mar 2016 13:06:24 -0600 Subject: [PATCH 032/507] No need to run queries to create the tables referenced by procedures. - The procedures drop and recreate these tables anyway, but if they don't exist the compilation fails. - Remove stray references to my username in comments. --- Oracle/PCORNetLoader_ora.sql | 102 ++++++++++++++++------------------- 1 file changed, 47 insertions(+), 55 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 60d5a8d..843a1d1 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1272,21 +1272,23 @@ whenever sqlerror continue; drop table priority; drop table location; -create table priority as ( - select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode PRIORITY - from i2b2fact - inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num - inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode - where c_fullname LIKE '\PCORI_MOD\PRIORITY\%' +create table priority ( + patient_num number(38,0), + encounter_num number(38,0), + provider_id varchar2(50 byte), + concept_cd varchar2(50 byte), + start_date date, + priority varchar2(50 byte) + ); + +create table location ( + patient_num number(38,0), + encounter_num number(38,0), + provider_id varchar2(50 byte), + concept_cd varchar2(50 byte), + start_date date, + result_loc varchar2(50 byte) ); - -create table location as ( - select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode RESULT_LOC - from i2b2fact - inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num - inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode - where c_fullname LIKE '\PCORI_MOD\RESULT_LOC\%' -); alter table blueheronmetadata.pcornet_lab add ( pcori_specimen_source varchar2(1000) -- arbitrary @@ -1441,44 +1443,35 @@ drop table quantity; drop table refills; drop table supply; -create table basis as ( - select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2fact basis - inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num - join pcornet_med basiscode - on basis.modifier_cd = basiscode.c_basecode - and basiscode.c_fullname like '\PCORI_MOD\RX_BASIS\%' - ); +create table basis ( + pcori_basecode varchar2(50 byte), + c_fullname varchar2(700 byte), + encounter_num number(38,0), + concept_cd varchar2(50 byte) + ) ; -create table freq as ( - select pcori_basecode,encounter_num,concept_cd from i2b2fact freq - inner join encounter enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num - join pcornet_med freqcode - on freq.modifier_cd = freqcode.c_basecode - and freqcode.c_fullname like '\PCORI_MOD\RX_FREQUENCY\%' +create table freq ( + pcori_basecode varchar2(50 byte), + encounter_num number(38,0), + concept_cd varchar2(50 byte) ); -create table quantity as ( - select nval_num,encounter_num,concept_cd from i2b2fact quantity - inner join encounter enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num - join pcornet_med quantitycode - on quantity.modifier_cd = quantitycode.c_basecode - and quantitycode.c_fullname like '\PCORI_MOD\RX_QUANTITY\' +create table quantity( + nval_num number(18,5), + encounter_num number(38,0), + concept_cd varchar2(50 byte) ); - -create table refills as ( - select nval_num,encounter_num,concept_cd from i2b2fact refills - inner join encounter enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num - join pcornet_med refillscode - on refills.modifier_cd = refillscode.c_basecode - and refillscode.c_fullname like '\PCORI_MOD\RX_REFILLS\' + +create table refills( + nval_num number(18,5), + encounter_num number(38,0), + concept_cd varchar2(50 byte) ); -create table supply as ( - select nval_num,encounter_num,concept_cd from i2b2fact supply - inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num - join pcornet_med supplycode - on supply.modifier_cd = supplycode.c_basecode - and supplycode.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\' +create table supply( + nval_num number(18,5), + encounter_num number(38,0), + concept_cd varchar2(50 byte) ); alter table blueheronmetadata.pcornet_med add ( @@ -1599,12 +1592,11 @@ Also, Error(57,57): PL/SQL: ORA-00904: "MO"."PCORI_NDC": invalid identifier whenever sqlerror continue; drop table amount; -create table amount as ( - select nval_num,encounter_num,concept_cd from i2b2fact amount - join pcornet_med amountcode - on amount.modifier_cd = amountcode.c_basecode - and amountcode.c_fullname like '\PCORI_MOD\RX_QUANTITY\' - ); +create table amount( + nval_num number(18,5), + encounter_num number(38,0), + concept_cd varchar2(50 byte) + ); alter table blueheronmetadata.pcornet_med add ( pcori_ndc varchar2(1000) -- arbitrary @@ -1756,9 +1748,9 @@ PCORNetLabResultCM; PCORNetPrescribing; /* ORA-04068: existing state of packages has been discarded -ORA-04065: not executed, altered or dropped stored procedure "NGRAHAM.PCORNETDISPENSING" -ORA-06508: PL/SQL: could not find program unit being called: "NGRAHAM.PCORNETDISPENSING" -ORA-06512: at "NGRAHAM.PCORNETLOADER", line 14 +ORA-04065: not executed, altered or dropped stored procedure "PCORNETDISPENSING" +ORA-06508: PL/SQL: could not find program unit being called: "PCORNETDISPENSING" +ORA-06512: at "PCORNETLOADER", line 14 ORA-06512: at line 2 04068. 00000 - "existing state of packages%s%s%s has been discarded" *Cause: One of errors 4060 - 4067 when attempt to execute a stored From febd503d5b63ff76973bf85eeb3762cffd6b803f Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Thu, 3 Mar 2016 17:07:57 -0600 Subject: [PATCH 033/507] First pass at translating modifiers - primary diagnosis (pdx). --- Oracle/pcornet_mapping.csv | 1 + Oracle/pcornet_mapping.sql | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 825ecac..b4046e0 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -1,4 +1,5 @@ pcori_path,local_path +\PCORI_MOD\PDX\P\,\Diagnosis\Clinical\PRIMARY_DX_YN\ \PCORI\DEMOGRAPHIC\BIOBANK_FLAG\Y\,\i2b2\Specimens\ \PCORI\DEMOGRAPHIC\HISPANIC\N\,"\i2b2\Demographics\Ethnicity\Non Hispanic, Latino or Spanish Origin\" \PCORI\DEMOGRAPHIC\HISPANIC\NI\,\i2b2\Demographics\Ethnicity\Not Recorded\ diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index cb2278a..e7bb81f 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -81,6 +81,21 @@ from terms_dxi td order by c_hlevel ; +-- Primary diagnosis modifier +update "&&i2b2_meta_schema".pcornet_diag pd +set pd.c_basecode = ( + select ht.c_basecode + from "&&i2b2_meta_schema"."&&terms_table" ht + join pcornet_mapping pm on pm.local_path = ht.c_fullname + where ht.c_tablename = 'MODIFIER_DIMENSION' and pm.pcori_path = pd.c_fullname + ) +where exists ( + select ht.c_basecode + from "&&i2b2_meta_schema"."&&terms_table" ht + join pcornet_mapping pm on pm.local_path = ht.c_fullname + where ht.c_tablename = 'MODIFIER_DIMENSION' and pm.pcori_path = pd.c_fullname + ); + commit; --select * From e5629c4434800e6dde308f8ec402e3244d4782a9 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Fri, 4 Mar 2016 08:59:49 -0600 Subject: [PATCH 034/507] Uncommented out RESULT_NUM data type fix in the LAB_RESULT_CM view. --- SAS/data_step_view_prep.sas | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SAS/data_step_view_prep.sas b/SAS/data_step_view_prep.sas index 98ac05f..fa76059 100644 --- a/SAS/data_step_view_prep.sas +++ b/SAS/data_step_view_prep.sas @@ -134,7 +134,7 @@ data sasdata.LAB_RESULT_CM / view=sasdata.LAB_RESULT_CM; rename = ( RESULT_TIME = _RESULT_TIME SPECIMEN_TIME = _SPECIMEN_TIME - /*RESULT_NUM = _RESULT_NUM*/ + RESULT_NUM = _RESULT_NUM ) ) ; @@ -147,8 +147,8 @@ data sasdata.LAB_RESULT_CM / view=sasdata.LAB_RESULT_CM; format SPECIMEN_TIME hhmm.; drop _SPECIMEN_TIME; - /*RESULT_NUM = put(_RESULT_NUM, best8.); - drop _RESULT_NUM;*/ + RESULT_NUM = put(_RESULT_NUM, best8.); + drop _RESULT_NUM; run; From 324ace93ec8a9048e7df89df05854b60e9cc7a70 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Fri, 4 Mar 2016 15:36:59 -0600 Subject: [PATCH 035/507] Script to populate the patient_dimension with ethnicity_cd from observation_facts. --- Oracle/update_ethnicity_pdim.sql | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 Oracle/update_ethnicity_pdim.sql diff --git a/Oracle/update_ethnicity_pdim.sql b/Oracle/update_ethnicity_pdim.sql new file mode 100644 index 0000000..dc48cfa --- /dev/null +++ b/Oracle/update_ethnicity_pdim.sql @@ -0,0 +1,53 @@ +/* Update the patient dimension to include column ETHNICITY_CD and populate it +with the appropriate PCORNet codes based on data in the observation_fact. + +The pcornet_mapping table contains columns that translate the PCORNet paths to +local paths. + +Example: +pcori_path,local_path +\PCORI\DEMOGRAPHIC\HISPANIC\Y\,\i2b2\Demographics\Ethnicity\Hispanic\ + +For ethnicity, it's assumed that the last field in the path is the PCORNet code. +In the example above, it's "Y". +*/ +select * from pcornet_mapping pm where 1=0; + +whenever sqlerror continue; +alter table "&&i2b2_data_schema".patient_dimension add ( + ETHNICITY_CD VARCHAR2(50 BYTE) + ); +drop table hispanic_patients; +whenever sqlerror exit; + +create table hispanic_patients as +with +hispanic_codes as ( + select + pm.pcori_path, replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code, pm.local_path, cd.concept_cd + from pcornet_mapping pm + join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' + where pm.pcori_path like '\PCORI\DEMOGRAPHIC\HISPANIC\%' + ) +-- Sometimes, we have more than one ethnicity fact for a given patient +select obs.patient_num, max(hc.pcori_code) pcori_code +from "&&i2b2_data_schema".observation_fact obs +join hispanic_codes hc on hc.concept_cd = obs.concept_cd +group by obs.patient_num +; + +update "&&i2b2_data_schema".patient_dimension pd +set ethnicity_cd = coalesce(( + select hp.pcori_code from hispanic_patients hp + where pd.patient_num = hp.patient_num + ), 'NI'); + +-- Make sure the update went as expected +select case when count(*) > 0 then 1/0 else 1 end ethnicity_update_match from ( + select * from ( + select pd.ethnicity_cd pd_code, hp.pcori_code hp_code + from "&&i2b2_data_schema".patient_dimension pd + left join hispanic_patients hp on hp.patient_num = pd.patient_num + ) + where pd_code != hp_code and not (pd_code = 'NI' and hp_code is null) + ); From 30a4bda38bfd70855a2a491acd127313769e3849 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 7 Mar 2016 12:00:34 -0600 Subject: [PATCH 036/507] Added failing test case for ethnicity/Hispanic. --- Oracle/cdm_transform_tests.sql | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 7c5227a..8a539d8 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -38,3 +38,34 @@ diff as ( from cdm cross join i2b2 ) select case when diff.pct > 10 then 1/0 else 1 end proc_pat_count_ok from diff; + + +/* Make sure we have roughly the same number of Hispanic patients in the CDM and +i2b2. +*/ +with +num_hispanic_cdm as ( + select count(*) qty from demographic where hispanic = 'Y' + ), +num_hispanic_i2b2 as ( + select count(*) qty from "&&i2b2_data_schema".patient_dimension where ethnicity_cd = 'Y' + ), +diff as ( + select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct + from num_hispanic_cdm cdm cross join num_hispanic_i2b2 i2b2 + ) +select case when diff.pct > 10 then 1/0 else 1 end hisp_y_pat_count_ok from diff; + +-- TODO: Consider trying to combine the Y and N tests as they are copy/paste +with +num_hispanic_cdm as ( + select count(*) qty from demographic where hispanic = 'N' + ), +num_hispanic_i2b2 as ( + select count(*) qty from "&&i2b2_data_schema".patient_dimension where ethnicity_cd = 'N' + ), +diff as ( + select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct + from num_hispanic_cdm cdm cross join num_hispanic_i2b2 i2b2 + ) +select case when diff.pct > 10 then 1/0 else 1 end hisp_n_pat_count_ok from diff; From 71d515bb8d669eb52c7e5b31ee84a36be29d7acc Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 7 Mar 2016 12:09:04 -0600 Subject: [PATCH 037/507] Ethnicity is now a column in the patient dimension so handle it the same way as race, sex, etc. - Update PCORNet ontology to include dimcodes, etc. for ethnicity. - Remove TODO about ethnicity --- Oracle/PCORNetLoader_ora.sql | 60 +++++++++---------------- Oracle/pcornet_mapping.sql | 15 +++++++ Oracle/pcornet_ontology_updates.csv | 7 +++ Oracle/pcornet_ontology_updates.ctl | 32 +++++++++++++ Oracle/pcornet_ontology_updates_ddl.sql | 29 ++++++++++++ Oracle/run-i2p-transform.sh | 4 ++ 6 files changed, 108 insertions(+), 39 deletions(-) create mode 100644 Oracle/pcornet_ontology_updates.csv create mode 100644 Oracle/pcornet_ontology_updates.ctl create mode 100644 Oracle/pcornet_ontology_updates_ddl.sql diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 843a1d1..0a9f4be 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -201,7 +201,7 @@ select 'RACE',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAP union select 'SEX',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' union -select 'HISPANIC',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC\Y%'; +select 'HISPANIC',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%'; begin @@ -796,26 +796,6 @@ END; / -/* TODO: Race vs Ethnicity - I think Eric at MCRF switched things around to use -the fact table rather than the RACE_CD in the patient dimension. -Refs: -https://informatics.gpcnetwork.org/trac/Project/ticket/191#comment:15 -http://listserv.kumc.edu/pipermail/gpc-dev/2016q1/002508.html (thread) - -Maybe something like the following, along with a change to the demographic -procedure? - - -update blueheronmetadata.pcornet_demo set - c_facttablecolumn = 'CONCEPT_CD', - c_tablename='CONCEPT_DIMENSION', - c_columnname = 'CONCEPT_PATH', - c_operator='LIKE', - c_dimcode='\i2b2\Demographics\Ethnicity\Hispanic' -where c_fullname = '\PCORI\DEMOGRAPHIC\HISPANIC\Y\'; -*/ - - create or replace procedure PCORNetDemographic as sqltext varchar2(4000); @@ -830,8 +810,8 @@ cursor getsql is ''''||race.pcori_basecode||''''|| ' from i2b2patient p '|| ' where lower(p.sex_cd) in ('||lower(sex.c_dimcode)||') '|| - ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| - ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') ' + ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| + ' and lower(nvl(p.ethnicity_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') ' from pcornet_demo race, pcornet_demo sex where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' and race.c_visualattributes like 'L%' @@ -848,12 +828,11 @@ select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPA ' from i2b2patient p '|| ' where lower(p.sex_cd) in ('||lower(sex.c_dimcode)||') '|| ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| - ' and lower(nvl(p.race_cd,''xx'')) in (select lower(code) from pcornet_codelist where codetype=''RACE'') '|| - ' and lower(nvl(p.race_cd,''xx'')) in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') ' + ' and lower(p.ethnicity_cd) in ('||lower(hisp.c_dimcode)||') ' from pcornet_demo race, pcornet_demo hisp, pcornet_demo sex where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' and race.c_visualattributes like 'L%' - and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC\Y%' + and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' and hisp.c_visualattributes like 'L%' and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' and sex.c_visualattributes like 'L%' @@ -868,7 +847,7 @@ union --2 S, nR, nH ' from i2b2patient p '|| ' where lower(nvl(p.sex_cd,''xx'')) in ('||lower(sex.c_dimcode)||') '|| ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') '|| - ' and lower(nvl(p.race_cd,''ni'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') ' + ' and lower(nvl(p.ethnicity_cd,''ni'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') ' from pcornet_demo sex where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' and sex.c_visualattributes like 'L%' @@ -883,7 +862,7 @@ union --3 -- nS,R, NH ' from i2b2patient p '|| ' where lower(nvl(p.sex_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''SEX'') '|| ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| - ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'')' + ' and lower(nvl(p.ethnicity_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'')' from pcornet_demo race where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' and race.c_visualattributes like 'L%' @@ -898,12 +877,11 @@ union --B -- nS,R, H ' from i2b2patient p '|| ' where lower(nvl(p.sex_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''SEX'') '|| ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| - ' and lower(nvl(p.race_cd,''xx'')) in (select lower(code) from pcornet_codelist where codetype=''RACE'') '|| - ' and lower(nvl(p.race_cd,''xx'')) in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'')' + ' and lower(p.ethnicity_cd) in ('||lower(hisp.c_dimcode)||') ' from pcornet_demo race,pcornet_demo hisp where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' and race.c_visualattributes like 'L%' - and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC\Y%' + and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' and hisp.c_visualattributes like 'L%' union --4 -- S, NR, H select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| @@ -911,28 +889,32 @@ union --4 -- S, NR, H ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| ''''||sex.pcori_basecode||''','|| - '''Y'','|| + ''''||hisp.pcori_basecode||''','|| '''NI'''|| ' from i2b2patient p '|| ' where lower(nvl(p.sex_cd,''NI'')) in ('||lower(sex.c_dimcode)||') '|| - ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') '|| - ' and lower(nvl(p.race_cd,''xx'')) in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') ' - from pcornet_demo sex + ' and lower(nvl(p.ethnicity_cd,''NI'')) in ('||lower(hisp.c_dimcode)||') '|| + ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') ' + from pcornet_demo sex, pcornet_demo hisp where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' and sex.c_visualattributes like 'L%' + and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' union --5 -- NS, NR, H select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| ' select ''5'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| '''NI'','|| - '''Y'','|| + ''''||hisp.pcori_basecode||''','|| '''NI'''|| ' from i2b2patient p '|| ' where lower(nvl(p.sex_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''SEX'') '|| ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') '|| - ' and lower(nvl(p.race_cd,''xx'')) in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'')' - from dual + ' and lower(nvl(p.ethnicity_cd,''NI'')) in ('||lower(hisp.c_dimcode)||') ' + from pcornet_demo hisp + where hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' union --6 -- NS, NR, nH select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| ' select ''6'',patient_num, '|| @@ -943,7 +925,7 @@ union --6 -- NS, NR, nH '''NI'''|| ' from i2b2patient p '|| ' where lower(nvl(p.sex_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''SEX'') '|| - ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') '|| + ' and lower(nvl(p.ethnicity_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') '|| ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') ' from dual; diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index e7bb81f..12a56c3 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -42,6 +42,21 @@ join "&&i2b2_meta_schema"."&&terms_table" i2b2 on i2b2.c_fullname=pcornet_mappin commit; +/* Updates to PCORNet demographic ontology + - Hispanic was added as a column, so updating code lists. + +TODO: Consider making a pull request to the SCILHS ontology with the changes. +See also: https://github.com/SCILHS/i2p-transform/issues/1 +*/ +delete +from "&&i2b2_meta_schema".pcornet_demo where c_fullname in ( + select c_fullname from pcornet_ontology_updates + where c_fullname like '\PCORI\DEMOGRAPHIC\%' + ); +insert into "&&i2b2_meta_schema".pcornet_demo ( + select * from pcornet_ontology_updates where c_fullname like '\PCORI\DEMOGRAPHIC\%' + ); + /* Replace PCORNet ICD9 diagnoses hierarchy with the local hierarchy filling in the pcornet_basecode with the expected values. diff --git a/Oracle/pcornet_ontology_updates.csv b/Oracle/pcornet_ontology_updates.csv new file mode 100644 index 0000000..9544604 --- /dev/null +++ b/Oracle/pcornet_ontology_updates.csv @@ -0,0 +1,7 @@ +"C_HLEVEL","C_FULLNAME","C_NAME","C_SYNONYM_CD","C_VISUALATTRIBUTES","C_TOTALNUM","C_BASECODE","C_METADATAXML","C_FACTTABLECOLUMN","C_TABLENAME","C_COLUMNNAME","C_COLUMNDATATYPE","C_OPERATOR","C_DIMCODE","C_COMMENT","C_TOOLTIP","M_APPLIED_PATH","UPDATE_DATE","DOWNLOAD_DATE","IMPORT_DATE","SOURCESYSTEM_CD","VALUETYPE_CD","M_EXCLUSION_CD","C_PATH","C_SYMBOL","PCORI_BASECODE" +2,"\PCORI\DEMOGRAPHIC\HISPANIC\","Ethnicity","N","CAE",,,"3.22014-05-09T11:10:26.266-04:00EnumNoUnknownYes ","concept_cd","CONCEPT_DIMENSION","concept_path","T","like","\PCORI\DEMOGRAPHIC\HISPANIC\",,"Ethnicity","@","2014-05-09 11:10:28","2014-05-09 11:10:28","2014-05-09 11:10:28","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\","HISPANIC", +3,"\PCORI\DEMOGRAPHIC\HISPANIC\NI\","No information","N","LAE",,"ETHNICITY:NI",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","N","IS","NULL",,"No information: A data field is present in the source system, but the source value is null or blank.","@","2014-06-02 19:40:30","2014-06-02 19:40:30","2014-06-02 19:40:30","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","NI","NI" +3,"\PCORI\DEMOGRAPHIC\HISPANIC\N\","Non-Hispanic","N","LIE",,"ETHNICITY:NOTHISPANIC",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'N'",,"Non-Hispanic","@","2014-05-09 11:12:04","2014-05-09 11:12:04","2014-05-09 11:12:04","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","N","N" +3,"\PCORI\DEMOGRAPHIC\HISPANIC\OT\","Other","N","LAE",,"ETHNICITY:OT",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'OT'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2014-06-02 19:44:16","2014-06-02 19:44:16","2014-06-02 19:44:16","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","OT","OT" +3,"\PCORI\DEMOGRAPHIC\HISPANIC\UN\","Unknown","N","LAE",,"ETHNICITY:UN",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'UN','u','unk'",,"Unknown: A data field is present in the source system, but the source value explicitly denotes an unknown value.","@","2014-06-02 19:43:08","2014-06-02 19:43:08","2014-06-02 19:43:08","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","UN","UN" +3,"\PCORI\DEMOGRAPHIC\HISPANIC\Y\","Hispanic","N","LAE",,"ETHNICITY:HISPANIC",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'Y','hib','his/black','his/white','hiw','hispanic'",,"A person of Cuban, Mexican, Puerto Rican, South or Central American, or other Spanish culture or origin, regardless of race. \ Hisptanic","@","2014-05-09 15:21:33","2014-05-09 11:11:12","2014-05-09 11:11:12","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","Y","Y" diff --git a/Oracle/pcornet_ontology_updates.ctl b/Oracle/pcornet_ontology_updates.ctl new file mode 100644 index 0000000..c0d966d --- /dev/null +++ b/Oracle/pcornet_ontology_updates.ctl @@ -0,0 +1,32 @@ +options (direct=true, errors=0, skip=1) +load data +truncate into table PCORNET_ONTOLOGY_UPDATES +fields terminated by ',' optionally enclosed by '"' +trailing nullcols( + C_HLEVEL, + C_FULLNAME, + C_NAME CHAR(2000), + C_SYNONYM_CD, + C_VISUALATTRIBUTES, + C_TOTALNUM, + C_BASECODE, + C_METADATAXML CHAR(100000), + C_FACTTABLECOLUMN, + C_TABLENAME, + C_COLUMNNAME, + C_COLUMNDATATYPE, + C_OPERATOR, + C_DIMCODE CHAR(700), + C_COMMENT, + C_TOOLTIP CHAR(900), + M_APPLIED_PATH, + UPDATE_DATE DATE 'YYYY-MM-DD HH24:MI:SS', + DOWNLOAD_DATE DATE 'YYYY-MM-DD HH24:MI:SS', + IMPORT_DATE DATE 'YYYY-MM-DD HH24:MI:SS', + SOURCESYSTEM_CD, + VALUETYPE_CD, + M_EXCLUSION_CD, + C_PATH, + C_SYMBOL, + PCORI_BASECODE + ) \ No newline at end of file diff --git a/Oracle/pcornet_ontology_updates_ddl.sql b/Oracle/pcornet_ontology_updates_ddl.sql new file mode 100644 index 0000000..82ec392 --- /dev/null +++ b/Oracle/pcornet_ontology_updates_ddl.sql @@ -0,0 +1,29 @@ +CREATE TABLE pcornet_ontology_updates ( + "C_HLEVEL" NUMBER(22,0) NOT NULL, + "C_FULLNAME" VARCHAR2(700) NOT NULL, + "C_NAME" VARCHAR2(2000) NOT NULL, + "C_SYNONYM_CD" CHAR(1) NOT NULL, + "C_VISUALATTRIBUTES" CHAR(3) NOT NULL, + "C_TOTALNUM" NUMBER(22,0) NULL, + "C_BASECODE" VARCHAR2(50) NULL, + "C_METADATAXML" CLOB NULL, + "C_FACTTABLECOLUMN" VARCHAR2(50) NOT NULL, + "C_TABLENAME" VARCHAR2(50) NOT NULL, + "C_COLUMNNAME" VARCHAR2(50) NOT NULL, + "C_COLUMNDATATYPE" VARCHAR2(50) NOT NULL, + "C_OPERATOR" VARCHAR2(10) NOT NULL, + "C_DIMCODE" VARCHAR2(700) NOT NULL, + "C_COMMENT" CLOB NULL, + "C_TOOLTIP" VARCHAR2(900) NULL, + "M_APPLIED_PATH" VARCHAR2(700) NOT NULL, + "UPDATE_DATE" DATE NOT NULL, + "DOWNLOAD_DATE" DATE NULL, + "IMPORT_DATE" DATE NULL, + "SOURCESYSTEM_CD" VARCHAR2(50) NULL, + "VALUETYPE_CD" VARCHAR2(50) NULL, + "M_EXCLUSION_CD" VARCHAR2(25) NULL, + "C_PATH" VARCHAR2(700) NULL, + "C_SYMBOL" VARCHAR2(50) NULL, + "PCORI_BASECODE" VARCHAR2(50) NULL +) +; diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 72089f5..636d690 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -24,13 +24,17 @@ set echo on; WHENEVER SQLERROR CONTINUE; drop table pcornet_mapping; +drop table pcornet_ontology_updates; WHENEVER SQLERROR EXIT SQL.SQLCODE; start pcornet_mapping_ddl.sql +start pcornet_ontology_updates_ddl.sql EOF ORACLE_SID=${sid} sqlldr ${pcornet_cdm_user}/${pcornet_cdm} control=pcornet_mapping.ctl data=pcornet_mapping.csv bad=pcornet_mapping.bad log=pcornet_mapping.log errors=0 +ORACLE_SID=${sid} sqlldr ${pcornet_cdm_user}/${pcornet_cdm} control=pcornet_ontology_updates.ctl data=pcornet_ontology_updates.csv bad=pcornet_ontology_updates.bad log=pcornet_ontology_updates.log errors=0 + # Insert local terms as leaves of the PCORNet terms and run the transform sqlplus /nolog < Date: Mon, 7 Mar 2016 13:35:03 -0600 Subject: [PATCH 038/507] Added CDM / HERON mapping for smoking field in VITAL. --- Oracle/cdm_transform_tests.sql | 6 ++++++ Oracle/pcornet_mapping.csv | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 7c5227a..5d69cc3 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -38,3 +38,9 @@ diff as ( from cdm cross join i2b2 ) select case when diff.pct > 10 then 1/0 else 1 end proc_pat_count_ok from diff; + +/* Test to make sure we have something about patient smoking tobacco use */ +with smokers as ( + select count(*) qty from vital where smoking!='NI' +) +select case when smokers.qty > 0 then 1 else 1/0 end smoker_count_ok from smokers; \ No newline at end of file diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index b4046e0..9471bb9 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -46,6 +46,14 @@ pcori_path,local_path \PCORI\VITAL\BP\SYSTOLIC\,\i2b2\Flowsheet\KU\GEN TMP CARD ECHO VITALS T:900426\KU GEN CARD GRP CV COMMON VITALS G:910000 I#2\BLOOD PRESSURE M:5 I#2\SYSTOLIC\ \PCORI\VITAL\ORIGINAL_BMI\,\i2b2\Flowsheet\MODEL\VITAL SIGNS COMPLEX T:31010\KU IP GRP ADULT HEIGHT/WEIGHT G:300220 I#4\KU IP ROW BMI M:301070 I#6\ \PCORI\VITAL\WT\,\i2b2\Flowsheet\KU\GEN CARD\IP/OP HEIGHT/WEIGHT T:900445\KU GEN CARD G IP/OP PROCEDURE HEIGHT/WEI G:900612 I#1\WEIGHT/SCALE M:14 I#2\ +\PCORI\VITAL\TOBACCO\SMOKING\YES\01\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Current Every Day Smoker\ +\PCORI\VITAL\TOBACCO\SMOKING\YES\02\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Current Some Day Smoker\ +\PCORI\VITAL\TOBACCO\SMOKING\03\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Former Smoker\ +\PCORI\VITAL\TOBACCO\SMOKING\04\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Never Smoker \ +\PCORI\VITAL\TOBACCO\SMOKING\YES\05\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Smoker, Current Status Unknown\ +\PCORI\VITAL\TOBACCO\SMOKING\06\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Unknown If Ever Smoked\ +\PCORI\VITAL\TOBACCO\SMOKING\YES\07\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Heavy Tobacco Smoker\ +\PCORI\VITAL\TOBACCO\SMOKING\YES\08\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Light Tobacco Smoker\ \PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\253\ \PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\09\ \PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\10\ From a846af25d9e41e37280d6c0425dd778556063ed6 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 7 Mar 2016 14:58:21 -0600 Subject: [PATCH 039/507] Add failing test cases for PDX and DX_SOURCE columns in the diagnosis table. --- Oracle/cdm_transform_tests.sql | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 8a539d8..455712a 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -69,3 +69,23 @@ diff as ( from num_hispanic_cdm cdm cross join num_hispanic_i2b2 i2b2 ) select case when diff.pct > 10 then 1/0 else 1 end hisp_n_pat_count_ok from diff; + +-- Make sure we have some diagnosis source information +select case when count(*) < 3 then 1/0 else 1 end a_few_dx_sources from ( + select distinct dx_source from diagnosis + ); + +-- Make sure we have valid DX_SOURCE values +select case when count(*) > 0 then 1/0 else 1 end valid_dx_sources from ( + select distinct dx_source from diagnosis where dx_source not in ('AD', 'DI', 'FI', 'IN', 'NI', 'UN', 'OT') + ); + +-- Make sure we have a couple principal diagnoses (PDX) +select case when count(*) < 2 then 1/0 else 1 end a_few_pdx_flags from ( + select distinct pdx from diagnosis + ); + +-- Make sure we have valid PDX values +select case when count(*) > 0 then 1/0 else 1 end valid_pdx_flags from ( + select distinct pdx from diagnosis where pdx not in ('P', 'S', 'X', 'NI', 'UN', 'OT') + ); From 37a5b95854df8d44204d2d1687c9750b8f38d0c0 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 7 Mar 2016 15:00:19 -0600 Subject: [PATCH 040/507] Add admit/discharge diagnosis - handle primary diagnosis at the same time. --- Oracle/pcornet_mapping.csv | 2 ++ Oracle/pcornet_mapping.sql | 52 ++++++++++++++++++++++++++++---------- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index b4046e0..4217cf7 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -1,5 +1,7 @@ pcori_path,local_path \PCORI_MOD\PDX\P\,\Diagnosis\Clinical\PRIMARY_DX_YN\ +\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\DI\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\ +\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\AD\,\Diagnosis\\modifier_dx\billing_diagnosis\admit\ \PCORI\DEMOGRAPHIC\BIOBANK_FLAG\Y\,\i2b2\Specimens\ \PCORI\DEMOGRAPHIC\HISPANIC\N\,"\i2b2\Demographics\Ethnicity\Non Hispanic, Latino or Spanish Origin\" \PCORI\DEMOGRAPHIC\HISPANIC\NI\,\i2b2\Demographics\Ethnicity\Not Recorded\ diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 12a56c3..f7b31c1 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -57,6 +57,13 @@ insert into "&&i2b2_meta_schema".pcornet_demo ( select * from pcornet_ontology_updates where c_fullname like '\PCORI\DEMOGRAPHIC\%' ); +/* Bugs in the SCILHS ontology +*/ +--https://github.com/SCILHS/scilhs-ontology/issues/11 +update "&&i2b2_meta_schema".pcornet_diag pd set pd.pcori_basecode = 'PDX:S' where c_fullname = '\PCORI_MOD\PDX\S\'; +update "&&i2b2_meta_schema".pcornet_diag pd set pd.pcori_basecode = 'PDX:P' where c_fullname = '\PCORI_MOD\PDX\P\'; +update "&&i2b2_meta_schema".pcornet_diag pd set pd.pcori_basecode = 'PDX:X' where c_fullname = '\PCORI_MOD\PDX\X\'; + /* Replace PCORNet ICD9 diagnoses hierarchy with the local hierarchy filling in the pcornet_basecode with the expected values. @@ -96,20 +103,37 @@ from terms_dxi td order by c_hlevel ; --- Primary diagnosis modifier -update "&&i2b2_meta_schema".pcornet_diag pd -set pd.c_basecode = ( - select ht.c_basecode - from "&&i2b2_meta_schema"."&&terms_table" ht - join pcornet_mapping pm on pm.local_path = ht.c_fullname - where ht.c_tablename = 'MODIFIER_DIMENSION' and pm.pcori_path = pd.c_fullname - ) -where exists ( - select ht.c_basecode - from "&&i2b2_meta_schema"."&&terms_table" ht - join pcornet_mapping pm on pm.local_path = ht.c_fullname - where ht.c_tablename = 'MODIFIER_DIMENSION' and pm.pcori_path = pd.c_fullname - ); + +-- Other diagnosis mappings such as modifiers for PDX, DX_SOURCE. +insert into "&&i2b2_meta_schema".PCORNET_DIAG +SELECT pcornet_diag.C_HLEVEL+1, + pcornet_diag.C_FULLNAME || i2b2.c_basecode || '\' as C_FULLNAME, + i2b2.c_basecode || ' ' || i2b2.c_name as C_NAME, + pcornet_diag.C_SYNONYM_CD, + pcornet_diag.C_VISUALATTRIBUTES, + pcornet_diag.C_TOTALNUM, + i2b2.c_basecode as C_BASECODE, + pcornet_diag.C_METADATAXML, + pcornet_diag.C_FACTTABLECOLUMN, + pcornet_diag.C_TABLENAME, + pcornet_diag.C_COLUMNNAME, + pcornet_diag.C_COLUMNDATATYPE, + pcornet_diag.C_OPERATOR, + pcornet_diag.C_FULLNAME || i2b2.c_basecode || '\' as C_DIMCODE, + pcornet_diag.C_COMMENT, + pcornet_diag.C_TOOLTIP, + pcornet_diag.M_APPLIED_PATH, + pcornet_diag.UPDATE_DATE, + pcornet_diag.DOWNLOAD_DATE, + pcornet_diag.IMPORT_DATE, + 'MAPPING' as SOURCESYSTEM_CD, + pcornet_diag.VALUETYPE_CD, + pcornet_diag.M_EXCLUSION_CD, + pcornet_diag.C_PATH, + pcornet_diag.C_SYMBOL, + pcornet_diag.PCORI_BASECODE +FROM "&&i2b2_meta_schema".pcornet_diag join pcornet_mapping on pcornet_mapping.PCORI_PATH = pcornet_diag.c_fullname and pcornet_mapping.local_path is not null +join "&&i2b2_meta_schema"."&&terms_table" i2b2 on i2b2.c_fullname like pcornet_mapping.local_path || '%'; commit; From 0793949a5eb9e696c078a507d8e40c316cb9bab9 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 08:33:19 -0600 Subject: [PATCH 041/507] Use GPC modifier "Billing Diagnosis - Discharge Primary" for principal discharge diagnosis flag (PDX). --- Oracle/pcornet_mapping.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 4217cf7..0579cc5 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -1,5 +1,5 @@ pcori_path,local_path -\PCORI_MOD\PDX\P\,\Diagnosis\Clinical\PRIMARY_DX_YN\ +\PCORI_MOD\PDX\P\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\discharge_primary\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\DI\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\AD\,\Diagnosis\\modifier_dx\billing_diagnosis\admit\ \PCORI\DEMOGRAPHIC\BIOBANK_FLAG\Y\,\i2b2\Specimens\ From 88db1cefb943a3519eaba0c3d479d2404cc1e77d Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 09:06:44 -0600 Subject: [PATCH 042/507] Added failing test case to make sure we have many different enrollment dates. --- Oracle/cdm_transform_tests.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 455712a..58c95c2 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -89,3 +89,17 @@ select case when count(*) < 2 then 1/0 else 1 end a_few_pdx_flags from ( select case when count(*) > 0 then 1/0 else 1 end valid_pdx_flags from ( select distinct pdx from diagnosis where pdx not in ('P', 'S', 'X', 'NI', 'UN', 'OT') ); + +-- Make sure that at least half of our enrollment records are distinct dates +select case when pct_distinct < 5 then 1/0 else 1 end many_enr_dates from ( + with half_enrs as ( + select round((select count(*)/2 from enrollment)) qty from dual + ), + distinct_date as ( + select count(qty) qty from ( + select distinct enr_start_date qty from enrollment + ) + ) + select round((distinct_date.qty/half_enrs.qty) * 100, 4) pct_distinct + from distinct_date cross join half_enrs + ); From e7a0b1b272de60e0ccd8cce357c3d35337054304 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 09:07:12 -0600 Subject: [PATCH 043/507] Use GPC definition of enrollment. --- Oracle/PCORNetLoader_ora.sql | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 0a9f4be..7d8ad52 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1230,11 +1230,25 @@ create or replace procedure PCORNetEnroll as begin INSERT INTO enrollment(PATID, ENR_START_DATE, ENR_END_DATE, CHART, ENR_BASIS) - select x.patient_num patid, case when l.patient_num is not null then l.period_start else enr_start end enr_start_date - , case when l.patient_num is not null then l.period_end when enr_end_end>enr_end then enr_end_end else enr_end end enr_end_date - , 'Y' chart, case when l.patient_num is not null then 'A' else 'E' end basis from - (select patient_num, min(start_date) enr_start,max(start_date) enr_end,max(end_date) enr_end_end from i2b2visit where patient_num in (select patid from demographic) group by patient_num) x - left outer join i2b2loyalty_patients l on l.patient_num=x.patient_num; +with pats_delta as ( + -- If only one visit, visit_delta_days will be 0 + select patient_num, max(start_date) - min(start_date) visit_delta_days + from i2b2visit + where start_date > add_months(sysdate, -36) + group by patient_num + ), +enrolled as ( + select distinct patient_num + from pats_delta + where visit_delta_days > 30 + ) +select + visit.patient_num patid, min(visit.start_date) enr_start_date, + max(visit.start_date) enr_end_date, 'Y' chart, 'A' enr_basis +from enrolled enr +join i2b2visit visit on enr.patient_num = visit.patient_num +group by visit.patient_num; + end PCORNetEnroll; / From f7eb1833aafabe3470f6cb8e9443473ff0fdace4 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 09:15:54 -0600 Subject: [PATCH 044/507] Base enrollment date test on percentage of all enrollment records rather than 1/2. --- Oracle/cdm_transform_tests.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 58c95c2..8c947c8 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -90,16 +90,16 @@ select case when count(*) > 0 then 1/0 else 1 end valid_pdx_flags from ( select distinct pdx from diagnosis where pdx not in ('P', 'S', 'X', 'NI', 'UN', 'OT') ); --- Make sure that at least half of our enrollment records are distinct dates +-- Make sure that there are several different enrollment dates select case when pct_distinct < 5 then 1/0 else 1 end many_enr_dates from ( - with half_enrs as ( - select round((select count(*)/2 from enrollment)) qty from dual + with all_enrs as ( + select count(*) qty from enrollment ), distinct_date as ( select count(qty) qty from ( select distinct enr_start_date qty from enrollment ) ) - select round((distinct_date.qty/half_enrs.qty) * 100, 4) pct_distinct - from distinct_date cross join half_enrs + select round((distinct_date.qty/all_enrs.qty) * 100, 4) pct_distinct + from distinct_date cross join all_enrs ); From 5a9068d1cfe1f6cb50746c8aa813aa403122482b Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 09:36:20 -0600 Subject: [PATCH 045/507] Failing test case making sure we have procedure dates. --- Oracle/cdm_transform_tests.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 8c947c8..4ddd8fc 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -103,3 +103,15 @@ select case when pct_distinct < 5 then 1/0 else 1 end many_enr_dates from ( select round((distinct_date.qty/all_enrs.qty) * 100, 4) pct_distinct from distinct_date cross join all_enrs ); + +-- Make sure most procedure dates are not null +select case when pct_not_null < 99 then 1/0 else 1 end some_px_dates_not_null from ( + with all_px as ( + select count(*) qty from procedures + ), + not_null as ( + select count(*) qty from procedures where px_date is not null + ) + select round((not_null.qty / all_px.qty) * 100, 4) pct_not_null + from not_null cross join all_px +); \ No newline at end of file From 9a867f6341340f2356621e018594b79eb486dd99 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 09:40:04 -0600 Subject: [PATCH 046/507] For procedures, use admit date from the encounter table and the fact date for px_date. --- Oracle/PCORNetLoader_ora.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 7d8ad52..2d69bbc 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1126,8 +1126,8 @@ end PCORNetCondition; create or replace procedure PCORNetProcedure as begin insert into procedures( - patid, encounterid, enc_type, admit_date, providerid, px, px_type) -select distinct fact.patient_num, enc.encounterid, enc.enc_type, fact.start_date, + patid, encounterid, enc_type, admit_date, px_date, providerid, px, px_type) +select distinct fact.patient_num, enc.encounterid, enc.enc_type, enc.admit_date, fact.start_date, fact.provider_id, SUBSTR(pr.pcori_basecode,INSTR(pr.pcori_basecode, ':')+1,11) px, SUBSTR(pr.c_fullname,18,2) pxtype from i2b2fact fact inner join encounter enc on enc.patid = fact.patient_num and enc.encounterid = fact.encounter_Num From 81a8515e9c740ed0f83ef158d1a6270f0080809e Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 09:53:45 -0600 Subject: [PATCH 047/507] Failing test to make sure we have some procedure source values. --- Oracle/cdm_transform_tests.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 4ddd8fc..e22e277 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -114,4 +114,9 @@ select case when pct_not_null < 99 then 1/0 else 1 end some_px_dates_not_null fr ) select round((not_null.qty / all_px.qty) * 100, 4) pct_not_null from not_null cross join all_px -); \ No newline at end of file +); + +-- Make sure we have some procedure sources (px_source) +select case when count(*) = 0 then 1/0 else 1 end have_px_sources from ( + select distinct px_source from procedures where px_source is not null + ); From e4c43961c3e9eb43ed6bbd199cabb3373f74256f Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 09:54:06 -0600 Subject: [PATCH 048/507] Hard-code px_source as "billing" for now. --- Oracle/PCORNetLoader_ora.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 2d69bbc..a65a84a 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1126,9 +1126,11 @@ end PCORNetCondition; create or replace procedure PCORNetProcedure as begin insert into procedures( - patid, encounterid, enc_type, admit_date, px_date, providerid, px, px_type) + patid, encounterid, enc_type, admit_date, px_date, providerid, px, px_type, px_source) select distinct fact.patient_num, enc.encounterid, enc.enc_type, enc.admit_date, fact.start_date, - fact.provider_id, SUBSTR(pr.pcori_basecode,INSTR(pr.pcori_basecode, ':')+1,11) px, SUBSTR(pr.c_fullname,18,2) pxtype + fact.provider_id, SUBSTR(pr.pcori_basecode,INSTR(pr.pcori_basecode, ':')+1,11) px, SUBSTR(pr.c_fullname,18,2) pxtype, + -- All are billing for now - see https://informatics.gpcnetwork.org/trac/Project/ticket/491 + 'BI' px_source from i2b2fact fact inner join encounter enc on enc.patid = fact.patient_num and enc.encounterid = fact.encounter_Num inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd From c8c8fd6a6a2eaa4b21afa92bd18d7afb245e6c1b Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 10:55:08 -0600 Subject: [PATCH 049/507] Avoid performance issues for now by commenting out PCORNetCondition. --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index a65a84a..2486200 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1738,7 +1738,7 @@ PCORNetHarvest; PCORNetDemographic; PCORNetEncounter; PCORNetDiagnosis; -PCORNetCondition; +-- TODO: Put this back - avoid performance issues for now: PCORNetCondition; PCORNetProcedure; PCORNetVital; PCORNetEnroll; From cdc24d1a7f6f0d2b884487878d2ee53143b2d409 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 8 Mar 2016 11:31:19 -0600 Subject: [PATCH 050/507] Improved vital smoking transform test. --- Oracle/cdm_transform_tests.sql | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 5d69cc3..3a38abf 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -40,7 +40,16 @@ diff as ( select case when diff.pct > 10 then 1/0 else 1 end proc_pat_count_ok from diff; /* Test to make sure we have something about patient smoking tobacco use */ -with smokers as ( - select count(*) qty from vital where smoking!='NI' +with snums as ( + select smoking cat, count(smoking) qty from vital group by smoking +), +tot as ( + select sum(qty) as cnt from snums +), +calc as ( + select snums.cat, (snums.qty/tot.cnt*100) pct, + case when (snums.qty/tot.cnt*100) > 1 then 1 else 0 end tst + from snums, tot + where snums.cat!='NI' ) -select case when smokers.qty > 0 then 1 else 1/0 end smoker_count_ok from smokers; \ No newline at end of file +select case when sum(calc.tst) < 3 then 1/0 else 1 end pass from calc; \ No newline at end of file From b131d8f91a3d7eac0aaf75ef9eabda961b24715b Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 11:37:33 -0600 Subject: [PATCH 051/507] Adding quick test to make sure that the ethnicity code column exists in the patient dimension. - Exit the load script on the first error. --- Oracle/run-i2p-transform.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 636d690..891cf52 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -1,3 +1,6 @@ +#!/bin/bash +set -e + # Expected environment variables (put there by Jenkins, etc.) # Database SID @@ -16,7 +19,21 @@ # All i2b2 terms - used for local path mapping #export terms_table= -# Create/Load the local path mapping table +# Make sure the ethnicity code has been added to the patient dimension +# See update_ethnicity_pdim.sql +sqlplus /nolog < Date: Tue, 8 Mar 2016 12:17:14 -0600 Subject: [PATCH 052/507] Index the Hispanic patients table by patient number for faster update. --- Oracle/update_ethnicity_pdim.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Oracle/update_ethnicity_pdim.sql b/Oracle/update_ethnicity_pdim.sql index dc48cfa..a37267e 100644 --- a/Oracle/update_ethnicity_pdim.sql +++ b/Oracle/update_ethnicity_pdim.sql @@ -36,6 +36,8 @@ join hispanic_codes hc on hc.concept_cd = obs.concept_cd group by obs.patient_num ; +create index hispanic_codes_patnum_idx on hispanic_patients(patient_num); + update "&&i2b2_data_schema".patient_dimension pd set ethnicity_cd = coalesce(( select hp.pcori_code from hispanic_patients hp From a726eb33b0a14d1365f8398ce403d5903e971443 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 8 Mar 2016 12:40:40 -0600 Subject: [PATCH 053/507] Added vital.tobacco path mapping and simple transform test. --- Oracle/cdm_transform_tests.sql | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 3a38abf..db83554 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -52,4 +52,16 @@ calc as ( from snums, tot where snums.cat!='NI' ) -select case when sum(calc.tst) < 3 then 1/0 else 1 end pass from calc; \ No newline at end of file +select case when sum(calc.tst) < 3 then 1/0 else 1 end pass from calc; + + +/* Test to make sure we have something about patient general tobacco use */ +with tobacco as ( + select count(*) qty from vital where tobacco!='NI' +) +select case when tobacco.qty > 0 then 1 else 1/0 end tobacco_count_ok from tobacco; + + + + + From 67b8231754a93831121d89609db27471327e52be Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 8 Mar 2016 12:46:07 -0600 Subject: [PATCH 054/507] Forgot to commit added vital.tobacco path mapping. --- Oracle/pcornet_mapping.csv | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 9471bb9..03e8e89 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -54,6 +54,11 @@ pcori_path,local_path \PCORI\VITAL\TOBACCO\SMOKING\06\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Unknown If Ever Smoked\ \PCORI\VITAL\TOBACCO\SMOKING\YES\07\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Heavy Tobacco Smoker\ \PCORI\VITAL\TOBACCO\SMOKING\YES\08\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Light Tobacco Smoker\ +\PCORI\VITAL\TOBACCO\02\01\,\i2b2\History\Social History\Tobacco Usage\Yes\ +\PCORI\VITAL\TOBACCO\02\02\,\i2b2\History\Social History\Tobacco Usage\Never\ +\PCORI\VITAL\TOBACCO\02\03\,\i2b2\History\Social History\Tobacco Usage\Quit\ +\PCORI\VITAL\TOBACCO\02\04\,\i2b2\History\Social History\Tobacco Usage\Passive\ +\PCORI\VITAL\TOBACCO\UN\06\,\i2b2\History\Social History\Tobacco Usage\Not Asked\ \PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\253\ \PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\09\ \PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\10\ From ab1571bffc5b0c6dd337bb6f3cf91a81e2894fd2 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 8 Mar 2016 13:39:24 -0600 Subject: [PATCH 055/507] Improved tobacco transform test. --- Oracle/cdm_transform_tests.sql | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index db83554..f14b4fc 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -52,14 +52,20 @@ calc as ( from snums, tot where snums.cat!='NI' ) -select case when sum(calc.tst) < 3 then 1/0 else 1 end pass from calc; +select case when sum(calc.tst) < 3 then 1/0 else 1 end smoking_count_ok from calc; /* Test to make sure we have something about patient general tobacco use */ -with tobacco as ( - select count(*) qty from vital where tobacco!='NI' +with tnums as ( + select tobacco cat, count(tobacco) qty from vital group by tobacco +), +tot as ( + select sum(qty) as cnt from tnums +), +calc as ( + select tnums.cat, (tnums.qty/tot.cnt*100) pct, case when (tnums.qty/tot.cnt*100) > 1 then 1 else 0 end tst from tnums, tot where tnums.cat!='NI' ) -select case when tobacco.qty > 0 then 1 else 1/0 end tobacco_count_ok from tobacco; +select case when sum(calc.tst) < 3 then 1/0 else 1 end pass from calc; From 5e654fc238b5ab9e37e54f3fee9916ca411b63da Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 15:24:28 -0600 Subject: [PATCH 056/507] Added failing test case for encounter type. --- Oracle/cdm_transform_tests.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 1c55de9..2f21245 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -125,3 +125,15 @@ with smokers as ( select count(*) qty from vital where smoking!='NI' ) select case when smokers.qty > 0 then 1 else 1/0 end smoker_count_ok from smokers; + +-- Make sure we have some encounter types +select case when pct_known < 20 then 1/0 else 1 end some_known_enc_types from ( + with all_enc as ( + select count(*) qty from encounter + ), + known_enc as ( + select count(*) qty from encounter where enc_type is not null and enc_type != 'UN' + ) + select round((known_enc.qty / all_enc.qty) * 100, 4) pct_known + from known_enc cross join all_enc + ); From 0c806a95af2f67b838a6a357c542dacfa71e384b Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 15:26:45 -0600 Subject: [PATCH 057/507] Added .sql to backfill the visit dimensions inout_cd column with encounter type codes. --- Oracle/update_inout_cd_vdim.sql | 49 +++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 Oracle/update_inout_cd_vdim.sql diff --git a/Oracle/update_inout_cd_vdim.sql b/Oracle/update_inout_cd_vdim.sql new file mode 100644 index 0000000..fce11e4 --- /dev/null +++ b/Oracle/update_inout_cd_vdim.sql @@ -0,0 +1,49 @@ +/* Update the visit dimension to populate the inout_cd with the appropriate +PCORNet encounter tyep codes based on data in the observation_fact. + +The pcornet_mapping table contains columns that translate the PCORNet paths to +local paths. + +Example: +pcori_path,local_path +\PCORI\ENCOUNTER\ENC_TYPE\AV\,\i2b2\Visit Details\ENC_TYPE\AV\ +*/ +select * from pcornet_mapping pm where 1=0; + +whenever sqlerror continue; +drop table enc_type; +whenever sqlerror exit; + +create table enc_type as +with +enc_type_codes as ( + select + pm.pcori_path, replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code, pm.local_path, cd.concept_cd + from pcornet_mapping pm + join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' + where pm.pcori_path like '\PCORI\ENCOUNTER\ENC_TYPE\%' + ) +-- TODO: Consider investigating why we have multiple encounter types for a single encounter +select obs.encounter_num, max(et.pcori_code) pcori_code +from "&&i2b2_data_schema".observation_fact obs +join enc_type_codes et on et.concept_cd = obs.concept_cd +group by obs.encounter_num +; + +create index enc_type_codes_encnum_idx on enc_type(encounter_num); + +update "&&i2b2_data_schema".visit_dimension vd +set inout_cd = coalesce(( + select et.pcori_code from enc_type et + where vd.encounter_num = et.encounter_num + ), 'UN'); + +-- Make sure the update went as expected +select case when count(*) > 0 then 1/0 else 1 end inout_cd_update_match from ( + select * from ( + select vd.inout_cd vd_code, et.pcori_code et_code + from "&&i2b2_data_schema".visit_dimension vd + left join enc_type et on et.encounter_num = vd.encounter_num + ) + where vd_code != vd_code and not (vd_code = 'UN' and vd_code is null) + ); From ac34c7e06dd68a2293ffb55aa1e9c0d5d655e71a Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 15:29:22 -0600 Subject: [PATCH 058/507] Update encounter type local path map. --- Oracle/pcornet_mapping.csv | 23 +++++------------------ Oracle/pcornet_mapping.sql | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index e591c93..573d646 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -9,24 +9,11 @@ pcori_path,local_path \PCORI\DEMOGRAPHIC\HISPANIC\Y\,\i2b2\Demographics\Ethnicity\Hispanic\ \PCORI\DEMOGRAPHIC\HISPANIC\Y\,\i2b2\Demographics\Ethnicity\Hispanic or Latino\ \PCORI\DEMOGRAPHIC\HISPANIC\Y\,"\i2b2\Demographics\Ethnicity\Hispanic, Latino or Spanish Origin\" -\PCORI\ENCOUNTER\ENC_TYPE\AV\,\i2b2\Visit Details\Encounter Type\102\ -\PCORI\ENCOUNTER\ENC_TYPE\AV\,\i2b2\Visit Details\Encounter Type\126\ -\PCORI\ENCOUNTER\ENC_TYPE\ED\,\i2b2\Visit Details\Encounter Type\103\ -\PCORI\ENCOUNTER\ENC_TYPE\IP\,\i2b2\Visit Details\Encounter Type\101\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\110\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\128\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\122\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\127\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\107\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\104\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\130\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\125\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\124\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\114\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\123\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\109\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\115\ -\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\Encounter Type\129\ +\PCORI\ENCOUNTER\ENC_TYPE\AV\,\i2b2\Visit Details\ENC_TYPE\AV\ +\PCORI\ENCOUNTER\ENC_TYPE\AV\,\i2b2\Visit Details\ENC_TYPE\AV\ +\PCORI\ENCOUNTER\ENC_TYPE\ED\,\i2b2\Visit Details\ENC_TYPE\ED\ +\PCORI\ENCOUNTER\ENC_TYPE\IP\,\i2b2\Visit Details\ENC_TYPE\IP\ +\PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\ENC_TYPE\OT\ \PCORI\DEMOGRAPHIC\SEX\F\,\i2b2\Demographics\Gender\Female\ \PCORI\DEMOGRAPHIC\SEX\M\,\i2b2\Demographics\Gender\Male\ \PCORI\DEMOGRAPHIC\SEX\NI\,\i2b2\Demographics\Gender\Unknown\Unknown-@\ diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index f7b31c1..625ee9c 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -40,6 +40,36 @@ SELECT PCORNET_VITAL.C_HLEVEL+1, FROM "&&i2b2_meta_schema".PCORNET_VITAL join pcornet_mapping on pcornet_mapping.PCORI_PATH=PCORNET_VITAL.c_fullname and pcornet_mapping.local_path is not null join "&&i2b2_meta_schema"."&&terms_table" i2b2 on i2b2.c_fullname=pcornet_mapping.local_path; +insert into "&&i2b2_meta_schema".PCORNET_ENC +SELECT PCORNET_ENC.C_HLEVEL+1, + PCORNET_ENC.C_FULLNAME || i2b2.c_name || '\' as C_FULLNAME, + i2b2.c_basecode || ' ' || i2b2.c_name as C_NAME, + PCORNET_ENC.C_SYNONYM_CD, + PCORNET_ENC.C_VISUALATTRIBUTES, + PCORNET_ENC.C_TOTALNUM, + i2b2.c_basecode as C_BASECODE, + PCORNET_ENC.C_METADATAXML, + PCORNET_ENC.C_FACTTABLECOLUMN, + PCORNET_ENC.C_TABLENAME, + PCORNET_ENC.C_COLUMNNAME, + PCORNET_ENC.C_COLUMNDATATYPE, + PCORNET_ENC.C_OPERATOR, + PCORNET_ENC.C_FULLNAME || i2b2.c_name || '\' as C_DIMCODE, + PCORNET_ENC.C_COMMENT, + PCORNET_ENC.C_TOOLTIP, + PCORNET_ENC.M_APPLIED_PATH, + PCORNET_ENC.UPDATE_DATE, + PCORNET_ENC.DOWNLOAD_DATE, + PCORNET_ENC.IMPORT_DATE, + 'MAPPING' as SOURCESYSTEM_CD, + PCORNET_ENC.VALUETYPE_CD, + PCORNET_ENC.M_EXCLUSION_CD, + PCORNET_ENC.C_PATH, + PCORNET_ENC.C_SYMBOL, + PCORNET_ENC.PCORI_BASECODE +FROM "&&i2b2_meta_schema".PCORNET_ENC join pcornet_mapping on pcornet_mapping.PCORI_PATH=PCORNET_ENC.c_fullname and pcornet_mapping.local_path is not null +join "&&i2b2_meta_schema"."&&terms_table" i2b2 on i2b2.c_fullname=pcornet_mapping.local_path; + commit; /* Updates to PCORNet demographic ontology From b1766ca23812fb7e94f43851c62628f154e37c53 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 8 Mar 2016 15:29:53 -0600 Subject: [PATCH 059/507] Added test for vital.tobacco_type transformation. --- Oracle/cdm_transform_tests.sql | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index f14b4fc..38140fc 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -68,6 +68,16 @@ calc as ( select case when sum(calc.tst) < 3 then 1/0 else 1 end pass from calc; - +/* Test to make sure we have something about tobacco use types */ +with ttnums as ( + select tobacco_type cat, count(tobacco) qty from vital group by tobacco_type order by cat +), +tot as ( + select sum(qty) as cnt from ttnums +), +calc as ( + select ttnums.cat, (ttnums.qty/tot.cnt*100) pct, case when (ttnums.qty/tot.cnt*100) > 1 then 1 else 0 end tst from ttnums, tot where ttnums.cat!='NI' +) +select case when sum(calc.tst) < 2 then 1/0 else 1 end pass from calc; From b281226415510387b493f00cea6560cefc83b932 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 8 Mar 2016 15:50:02 -0600 Subject: [PATCH 060/507] Add test to make sure the visit_dimension inout_cd has been populated before beginning the transform. --- Oracle/run-i2p-transform.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 891cf52..29092b5 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -19,8 +19,6 @@ set -e # All i2b2 terms - used for local path mapping #export terms_table= -# Make sure the ethnicity code has been added to the patient dimension -# See update_ethnicity_pdim.sql sqlplus /nolog < Date: Wed, 9 Mar 2016 16:38:00 -0600 Subject: [PATCH 061/507] Load local mapping tables before testing for ethnicity_cd, etc. - This is helpful because the mapping table is used in the SQL to update ethnicity_cd. --- Oracle/run-i2p-transform.sh | 38 +++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 29092b5..eaf01c0 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -19,6 +19,26 @@ set -e # All i2b2 terms - used for local path mapping #export terms_table= +# Create/Load the local path mapping and ontology update tables +sqlplus /nolog < Date: Thu, 10 Mar 2016 08:43:49 -0600 Subject: [PATCH 062/507] Replaced hard-coded schema references with variable. --- Oracle/PCORNetLoader_ora.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 2486200..7eae7d5 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1288,7 +1288,7 @@ create table location ( result_loc varchar2(50 byte) ); -alter table blueheronmetadata.pcornet_lab add ( +alter table "&&i2b2_meta_schema".pcornet_lab add ( pcori_specimen_source varchar2(1000) -- arbitrary ); whenever sqlerror exit; @@ -1472,7 +1472,7 @@ create table supply( concept_cd varchar2(50 byte) ); -alter table blueheronmetadata.pcornet_med add ( +alter table "&&i2b2_meta_schema".pcornet_med add ( pcori_cui varchar2(1000) -- arbitrary ); whenever sqlerror exit; @@ -1596,7 +1596,7 @@ create table amount( concept_cd varchar2(50 byte) ); -alter table blueheronmetadata.pcornet_med add ( +alter table "&&i2b2_meta_schema".pcornet_med add ( pcori_ndc varchar2(1000) -- arbitrary ); whenever sqlerror exit; From d5bdf6240bf8b775276ac0f40d9672d3511078be Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 10 Mar 2016 11:26:40 -0600 Subject: [PATCH 063/507] Refactored SAS data step view code. Extracted config into seperate file. --- SAS/configuration.example.sas | 38 ++++++++++++ SAS/data_step_view_inspect.sas | 108 +++++++++++++++++++++++++++++++++ SAS/data_step_view_prep.sas | 51 +++++----------- 3 files changed, 160 insertions(+), 37 deletions(-) create mode 100644 SAS/configuration.example.sas create mode 100644 SAS/data_step_view_inspect.sas diff --git a/SAS/configuration.example.sas b/SAS/configuration.example.sas new file mode 100644 index 0000000..f2f045f --- /dev/null +++ b/SAS/configuration.example.sas @@ -0,0 +1,38 @@ +/******************************************************************************* +* Oracle database connection and SAS data step view library configuration. +* +* For use in generating and inspecting PCORnet CMDv3 SAS data step views. +* +* Copy or rename this file to configuration.sas and replace the following +* placeholders with locally relavent values before running any of the associated +* SAS scripts which include configuration.sas: +* - ORC_SCHEMA +* - ORC_USERNAME +* - ORC_PASSWORD +* - OCR_HOSTNAME +* - ORC_SID +* - VIEW_LIB_PATH (e.x. C:/path/to/desired/data/step/view/directory/) +* +*******************************************************************************/ + + +***************************************************************; +* Oracle database connection information +***************************************************************; +libname oracdata oracle schema=ORC_SCHEMA user=ORC_USERNAME pw=ORC_PASSWORD + path="(DESCRIPTION = + (ADDRESS = + (PROTOCOL = TCP) + (HOST = ORC_HOSTNAME) + (PORT = 1521) + ) + (CONNECT_DATA = + (SID = ORC_SID) + ) + )"; + + +***************************************************************; +* Data step view location +***************************************************************; +libname sasdata 'VIEW_LIB_PATH'; diff --git a/SAS/data_step_view_inspect.sas b/SAS/data_step_view_inspect.sas new file mode 100644 index 0000000..be96d93 --- /dev/null +++ b/SAS/data_step_view_inspect.sas @@ -0,0 +1,108 @@ +/******************************************************************************* +* Inspect the PCORnet CDMv3 data step views created by data_step_view_prep.sas. +* Runs the SAS content procedure over and gets the first ten records from each +* of the data step views. +*******************************************************************************/ + + +***************************************************************; +* Clear SAS result buffer +***************************************************************; +ODS HTML CLOSE; +ODS HTML; + + +***************************************************************; +* Include configurable SAS libraies +***************************************************************; +%include './configuration.sas'; + + +proc contents data=sasdata.DEMOGRAPHIC; +run; +proc print data=sasdata.DEMOGRAPHIC (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.ENROLLMENT; +run; +proc print data=sasdata.ENROLLMENT (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.ENCOUNTER; +run; +proc print data=sasdata.ENCOUNTER (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.DIAGNOSIS; +run; +proc print data=sasdata.DIAGNOSIS (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.PROCEDURES; +run; +proc print data=sasdata.PROCEDURES (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.VITAL; +run; +proc print data=sasdata.VITAL (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.DISPENSING; +run; +proc print data=sasdata.DISPENSING (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.LAB_RESULT_CM; +run; +proc print data=sasdata.LAB_RESULT_CM (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.CONDITION; +run; +proc print data=sasdata.CONDITION (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.PRO_CM; +run; +proc print data=sasdata.PRO_CM (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.PRESCRIBING; +run; +proc print data=sasdata.PRESCRIBING (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.PCORNET_TRIAL; +run; +proc print data=sasdata.PCORNET_TRIAL (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.DEATH; +run; +proc print data=sasdata.DEATH (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.DEATH_CAUSE; +run; +proc print data=sasdata.DEATH_CAUSE (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.HARVEST; +run; +proc print data=sasdata.HARVEST (firstobs=1 obs=10); +run; diff --git a/SAS/data_step_view_prep.sas b/SAS/data_step_view_prep.sas index fa76059..bb2f2a2 100644 --- a/SAS/data_step_view_prep.sas +++ b/SAS/data_step_view_prep.sas @@ -1,39 +1,25 @@ /******************************************************************************* -* Generate the data step views required by the PRCOnet Diagnostic Query -* (diagnostic.sas). -* -* Provided the following values below before execution: -* - ORC_SCHEMA -* - ORC_USERNAME -* - ORC_PASSWORD -* - OCR_HOSTNAME -* - ORC_SID -* - VIEW_LIB_PATH -* +* Generate the PCORnet CDMv3 data step views required by PCORnet SAS queries, +* providing the required data type transformations where needed. *******************************************************************************/ + +***************************************************************; +* Clear SAS result buffer +***************************************************************; ODS HTML CLOSE; ODS HTML; -libname oracdata oracle schema=ORC_SCEMA user=ORC_USERNAME pw=ORC_PASSWORD - path="(DESCRIPTION = - (ADDRESS = - (PROTOCOL = TCP) - (HOST = ORC_HOSTNAME) - (PORT = 1521) - ) - (CONNECT_DATA = - (SID = ORC_SID) - ) - )"; - -libname sasdata 'VIEW_LIB_PATH'; + +***************************************************************; +* Include configurable SAS libraies +***************************************************************; +%include './configuration.sas'; ***************************************************************; * Create data step view for DEMOGRAPHIC ***************************************************************; - data sasdata.DEMOGRAPHIC / view=sasdata.DEMOGRAPHIC; set oracdata.DEMOGRAPHIC( rename = ( @@ -42,6 +28,9 @@ data sasdata.DEMOGRAPHIC / view=sasdata.DEMOGRAPHIC; ) ; + BIRTH_DATE = datepart(BIRTH_DATE); + format BIRTH_DATE mmddyy10.; + BIRTH_TIME = input(_BIRTH_TIME, hhmmss.); format BIRTH_TIME hhmm.; drop _BIRTH_TIME; @@ -51,7 +40,6 @@ run; ***************************************************************; * Create data step view for ENROLLMENT ***************************************************************; - data sasdata.ENROLLMENT / view=sasdata.ENROLLMENT; set oracdata.ENROLLMENT; run; @@ -60,7 +48,6 @@ run; ***************************************************************; * Create data step view for ENCOUNTER ***************************************************************; - data sasdata.ENCOUNTER / view=sasdata.ENCOUNTER; set oracdata.ENCOUNTER( rename = ( @@ -83,7 +70,6 @@ run; ***************************************************************; * Create data step view for DIAGNOSIS ***************************************************************; - data sasdata.DIAGNOSIS / view=sasdata.DIAGNOSIS; set oracdata.DIAGNOSIS; run; @@ -119,7 +105,6 @@ run; ***************************************************************; * Create data step view for DISPENSING ***************************************************************; - data sasdata.DISPENSING / view=sasdata.DISPENSING; set oracdata.DISPENSING; run; @@ -128,7 +113,6 @@ run; ***************************************************************; * Create data step view for LAB_RESULT_CM ***************************************************************; - data sasdata.LAB_RESULT_CM / view=sasdata.LAB_RESULT_CM; set oracdata.LAB_RESULT_CM( rename = ( @@ -155,7 +139,6 @@ run; ***************************************************************; * Create data step view for CONDITION ***************************************************************; - data sasdata.CONDITION / view=sasdata.CONDITION; set oracdata.CONDITION; run; @@ -164,7 +147,6 @@ run; ***************************************************************; * Create data step view for PRO_CM ***************************************************************; - data sasdata.PRO_CM / view=sasdata.PRO_CM; set oracdata.PRO_CM( rename = ( @@ -182,7 +164,6 @@ run; ***************************************************************; * Create data step view for PRESCRIBING ***************************************************************; - data sasdata.PRESCRIBING / view=sasdata.PRESCRIBING; set oracdata.PRESCRIBING( rename = ( @@ -200,7 +181,6 @@ run; ***************************************************************; * Create data step view for PCORNET_TRIAL ***************************************************************; - data sasdata.PCORNET_TRIAL / view=sasdata.PCORNET_TRIAL; set oracdata.PCORNET_TRIAL; run; @@ -209,7 +189,6 @@ run; ***************************************************************; * Create data step view for DEATH ***************************************************************; - data sasdata.DEATH / view=sasdata.DEATH; set oracdata.DEATH; run; @@ -218,7 +197,6 @@ run; ***************************************************************; * Create data step view for DEATH_CAUSE ***************************************************************; - data sasdata.DEATH_CAUSE / view=sasdata.DEATH_CAUSE; set oracdata.DEATH_CAUSE; run; @@ -227,7 +205,6 @@ run; ***************************************************************; * Create data step view for HARVEST ***************************************************************; - data sasdata.HARVEST / view=sasdata.HARVEST; set oracdata.HARVEST; run; From 55291031020915598ec0b491179ab133d4a83037 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Fri, 11 Mar 2016 08:55:38 -0600 Subject: [PATCH 064/507] Added data type transform for all CDMv3 date fields and improved config include. --- SAS/data_step_view_inspect.sas | 6 +- SAS/data_step_view_prep.sas | 120 ++++++++++++++++++++++++++++++++- 2 files changed, 122 insertions(+), 4 deletions(-) diff --git a/SAS/data_step_view_inspect.sas b/SAS/data_step_view_inspect.sas index be96d93..6c42b27 100644 --- a/SAS/data_step_view_inspect.sas +++ b/SAS/data_step_view_inspect.sas @@ -15,7 +15,11 @@ ODS HTML; ***************************************************************; * Include configurable SAS libraies ***************************************************************; -%include './configuration.sas'; +%let fpath=%sysget(SAS_EXECFILEPATH); +%let fname=%sysget(SAS_EXECFILENAME); +%let path= %sysfunc(tranwrd(&fpath,&fname,'')); +%put &path; +%include '&path/configuration.sas'; proc contents data=sasdata.DEMOGRAPHIC; diff --git a/SAS/data_step_view_prep.sas b/SAS/data_step_view_prep.sas index bb2f2a2..0187957 100644 --- a/SAS/data_step_view_prep.sas +++ b/SAS/data_step_view_prep.sas @@ -14,7 +14,11 @@ ODS HTML; ***************************************************************; * Include configurable SAS libraies ***************************************************************; -%include './configuration.sas'; +%let fpath=%sysget(SAS_EXECFILEPATH); +%let fname=%sysget(SAS_EXECFILENAME); +%let path= %sysfunc(tranwrd(&fpath,&fname,'')); +%put &path; +%include '&path/configuration.sas'; ***************************************************************; @@ -42,6 +46,13 @@ run; ***************************************************************; data sasdata.ENROLLMENT / view=sasdata.ENROLLMENT; set oracdata.ENROLLMENT; + + ENR_START_DATE = datepart(ENR_START_DATE); + format ENR_START_DATE mmddyy10.; + + ENR_END_DATE = datepart(ENR_END_DATE); + format ENR_END_DATE mmddyy10.; + run; @@ -57,10 +68,16 @@ data sasdata.ENCOUNTER / view=sasdata.ENCOUNTER; ) ; + ADMIT_DATE = datepart(ADMIT_DATE); + format ADMIT_DATE mmddyy10.; + ADMIT_TIME = input(_ADMIT_TIME, hhmmss.); format ADMIT_TIME hhmm.; drop _ADMIT_TIME; + DISCHARGE_DATE = datepart(DISCHARGE_DATE); + format DISCHARGE_DATE mmddyy10.; + DISCHARGE_TIME = input(_DISCHARGE_TIME, hhmmss.); format DISCHARGE_TIME hhmm.; drop _DISCHARGE_TIME; @@ -72,22 +89,29 @@ run; ***************************************************************; data sasdata.DIAGNOSIS / view=sasdata.DIAGNOSIS; set oracdata.DIAGNOSIS; + + ADMIT_DATE = datepart(ADMIT_DATE); + format ADMIT_DATE mmddyy10.; run; ***************************************************************; * Create data step view for PROCEDURES ***************************************************************; - data sasdata.PROCEDURES / view=sasdata.PROCEDURES; set oracdata.PROCEDURES; + + ADMIT_DATE = datepart(ADMIT_DATE); + format ADMIT_DATE mmddyy10.; + + PX_DATE = datepart(PX_DATE); + format PX_DATE mmddyy10.; run; ***************************************************************; * Create data step view for VITAL ***************************************************************; - data sasdata.VITAL / view=sasdata.VITAL; set oracdata.VITAL( rename = ( @@ -96,6 +120,9 @@ data sasdata.VITAL / view=sasdata.VITAL; ) ; + MEASURE_DATE = datepart(MEASURE_DATE); + format MEASURE_DATE mmddyy10.; + MEASURE_TIME = input(_MEASURE_TIME, hhmmss.); format MEASURE_TIME hhmm.; drop _MEASURE_TIME; @@ -107,6 +134,9 @@ run; ***************************************************************; data sasdata.DISPENSING / view=sasdata.DISPENSING; set oracdata.DISPENSING; + + DISPENSE_DATE = datepart(DISPENSE_DATE); + format DISPENSE_DATE mmddyy10.; run; @@ -122,10 +152,19 @@ data sasdata.LAB_RESULT_CM / view=sasdata.LAB_RESULT_CM; ) ) ; + + LAB_ORDER_DATE = datepart(LAB_ORDER_DATE); + format LAB_ORDER_DATE mmddyy10.; + + RESULT_DATE = datepart(RESULT_DATE); + format RESULT_DATE mmddyy10.; RESULT_TIME = input(_RESULT_TIME, hhmmss.); format RESULT_TIME hhmm.; drop _RESULT_TIME; + + SPECIMEN_DATE = datepart(SPECIMEN_DATE); + format SPECIMEN_DATE mmddyy10.; SPECIMEN_TIME = input(_SPECIMEN_TIME, hhmmss.); format SPECIMEN_TIME hhmm.; @@ -141,6 +180,15 @@ run; ***************************************************************; data sasdata.CONDITION / view=sasdata.CONDITION; set oracdata.CONDITION; + + REPORT_DATE = datepart(REPORT_DATE); + format REPORT_DATE mmddyy10.; + + RESOLVE_DATE = datepart(RESOLVE_DATE); + format RESOLVE_DATE mmddyy10.; + + ONSET_DATE = datepart(ONSET_DATE); + format ONSET_DATE mmddyy10.; run; @@ -155,6 +203,9 @@ data sasdata.PRO_CM / view=sasdata.PRO_CM; ) ; + PRO_DATE = datepart(PRO_DATE); + format PRO_DATE mmddyy10.; + PRO_TIME = input(_PRO_TIME, hhmmss.); format PRO_TIME hhmm.; drop _PRO_TIME; @@ -172,9 +223,18 @@ data sasdata.PRESCRIBING / view=sasdata.PRESCRIBING; ) ; + RX_ORDER_DATE = datepart(RX_ORDER_DATE); + format RX_ORDER_DATE mmddyy10.; + RX_ORDER_TIME = input(_RX_ORDER_TIME, hhmmss.); format RX_ORDER_TIME hhmm.; drop _RX_ORDER_TIME; + + RX_START_DATE = datepart(RX_START_DATE); + format RX_START_DATE mmddyy10.; + + RX_END_DATE = datepart(RX_END_DATE); + format RX_END_DATE mmddyy10.; run; @@ -183,6 +243,15 @@ run; ***************************************************************; data sasdata.PCORNET_TRIAL / view=sasdata.PCORNET_TRIAL; set oracdata.PCORNET_TRIAL; + + TRIAL_ENROLL_DATE = datepart(TRIAL_ENROLL_DATE); + format TRIAL_ENROLL_DATE mmddyy10.; + + TRIAL_END_DATE = datepart(TRIAL_END_DATE); + format TRIAL_END_DATE mmddyy10.; + + TRIAL_WITHDRAW_DATE = datepart(TRIAL_WITHDRAW_DATE); + format TRIAL_WITHDRAW_DATE mmddyy10.; run; @@ -191,6 +260,9 @@ run; ***************************************************************; data sasdata.DEATH / view=sasdata.DEATH; set oracdata.DEATH; + + DEATH_DATE = datepart(DEATH_DATE); + format DEATH_DATE mmddyy10.; run; @@ -207,4 +279,46 @@ run; ***************************************************************; data sasdata.HARVEST / view=sasdata.HARVEST; set oracdata.HARVEST; + + REFRESH_DEMOGRAPHIC_DATE = datepart(REFRESH_DEMOGRAPHIC_DATE); + format REFRESH_DEMOGRAPHIC_DATE mmddyy10.; + + REFRESH_ENROLLMENT_DATE = datepart(REFRESH_ENROLLMENT_DATE); + format REFRESH_ENROLLMENT_DATE mmddyy10.; + + REFRESH_ENCOUNTER_DATE = datepart(REFRESH_ENCOUNTER_DATE); + format REFRESH_ENCOUNTER_DATE mmddyy10.; + + REFRESH_DIAGNOSIS_DATE = datepart(REFRESH_DIAGNOSIS_DATE); + format REFRESH_DIAGNOSIS_DATE mmddyy10.; + + REFRESH_PROCEDURES_DATE = datepart(REFRESH_PROCEDURES_DATE); + format REFRESH_PROCEDURES_DATE mmddyy10.; + + REFRESH_VITAL_DATE = datepart(REFRESH_VITAL_DATE); + format REFRESH_VITAL_DATE mmddyy10.; + + REFRESH_DISPENSING_DATE = datepart(REFRESH_DISPENSING_DATE); + format REFRESH_DISPENSING_DATE mmddyy10.; + + REFRESH_LAB_RESULT_CM_DATE = datepart(REFRESH_LAB_RESULT_CM_DATE); + format REFRESH_LAB_RESULT_CM_DATE mmddyy10.; + + REFRESH_CONDITION_DATE = datepart(REFRESH_CONDITION_DATE); + format REFRESH_CONDITION_DATE mmddyy10.; + + REFRESH_PRO_CM_DATE = datepart(REFRESH_PRO_CM_DATE); + format REFRESH_PRO_CM_DATE mmddyy10.; + + REFRESH_PRESCRIBING_DATE = datepart(REFRESH_PRESCRIBING_DATE); + format REFRESH_PRESCRIBING_DATE mmddyy10.; + + REFRESH_PCORNET_TRIAL_DATE = datepart(REFRESH_PCORNET_TRIAL_DATE); + format REFRESH_PCORNET_TRIAL_DATE mmddyy10.; + + REFRESH_DEATH_DATE = datepart(REFRESH_DEATH_DATE); + format REFRESH_DEATH_DATE mmddyy10.; + + REFRESH_DEATH_CAUSE_DATE = datepart(REFRESH_DEATH_CAUSE_DATE); + format REFRESH_DEATH_CAUSE_DATE mmddyy10.; run; From 35d9be1467a34d2267ef8551be62151f0e9fecdd Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 11 Mar 2016 13:17:10 -0600 Subject: [PATCH 065/507] get discharge_date on ~10% of relevant encounters based on `UHC|LOS:1` - update i2b2 visit_dimension, then pcornet encounter --- Oracle/heron_encounter_style.sql | 168 +++++++++++++++++++++++++++++++ Oracle/run-i2p-transform.sh | 2 +- Oracle/update_inout_cd_vdim.sql | 49 --------- 3 files changed, 169 insertions(+), 50 deletions(-) create mode 100644 Oracle/heron_encounter_style.sql delete mode 100644 Oracle/update_inout_cd_vdim.sql diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql new file mode 100644 index 0000000..8d33eb3 --- /dev/null +++ b/Oracle/heron_encounter_style.sql @@ -0,0 +1,168 @@ +/** heron_encounter_style -- apply HERON / KUMC / GPC encounter conventions + +copyright (c) 2016 University of Kansas Medical Center +availble under the i2b2 Software License (aka MPL?) + +*/ + +/** Index patid, encounterid in hopes of speeding up exploration. + +alter session set current_schema = pcornet_cdm; + +create unique index demographic_patid on demographic (patid); +create unique index encounter_id on encounter (encounterid); + +create index encounter_patid on encounter (patid); +create index diagnosis_patid on diagnosis (patid); +create index diagnosis_encid on diagnosis (encounterid); +create index procedures_patid on procedures (patid); +create index procedures_encid on procedures (encounterid); +create index vital_patid on vital (patid); + */ + + +/** get length_of_stay, visit end_date from fact table + +TODO: push this back into HERON ETL +*/ +merge into "&&i2b2_data_schema".visit_dimension vd +using ( + select obs.encounter_num, obs.patient_num + , obs.nval_num length_of_stay + , obs.end_date + from "&&i2b2_data_schema".observation_fact obs + where obs.concept_cd = 'UHC|LOS:1' -- TODO: parameterize +) obs +on ( obs.encounter_num = vd.encounter_num + and obs.patient_num = vd.patient_num) +when matched then update + set vd.length_of_stay = obs.length_of_stay + , vd.end_date = obs.end_date +; +-- 194,328 rows merged. + + +merge into encounter pe +using "&&i2b2_data_schema".visit_dimension vd + on ( pe.encounterid = vd.encounter_num + and vd.end_date is not null) +when matched then update set pe.discharge_date = vd.end_date; +-- 165,292 rows merged. + + +/* Check that we have encounter.discharge_date for selected visit types. + +"Discharge date. Should be populated for all +Inpatient Hospital Stay (IP) and Non-Acute +Institutional Stay (IS) encounter types." + -- PCORnet CDM v3 +*/ +with enc_agg as ( + select count(*) enc_qty from encounter + where enc_type in ('IP', 'IS') + ) +select count(*), round(count(*) / enc_qty * 100, 1) pct +from encounter cross join enc_agg +where enc_type in ('IP', 'IS') +and discharge_date is null +group by enc_qty +; + + + +/** TODO: chase down ~80% unknown enc_type */ +with enc_agg as ( +select count(*) qty from encounter) +select count(*) encounter_qty + , round(count(*) / enc_agg.qty * 100, 1) pct + , enc_type +from encounter enc +cross join enc_agg +where exists ( + select admit_date from diagnosis dx where enc.patid = dx.patid) +or exists ( + select admit_date from procedures px where enc.patid = px.patid) +group by enc_type, enc_agg.qty +; + +/* TODO: get encounter type from source system? */ +select count(*), sourcesystem_cd +from nightherondata.visit_dimension +group by sourcesystem_cd; +*/ + +/* TODO: get encounter type from place of service? */ +select obs.encounter_num, obs.patient_num, obs.start_date, obs.end_date + , con.concept_cd, con.name_char place_of_service + , obs.sourcesystem_cd +from "&&i2b2_data_schema".observation_fact obs +join "&&i2b2_data_schema".concept_dimension con on con.concept_cd = obs.concept_cd +join "&&i2b2_data_schema".visit_dimension v on v.encounter_num = obs.encounter_num +where con.concept_path like '\i2b2\Visit Details\Place of Service (IDX)\%' +and v.inout_cd = 'OT' +; + + +/* Update the visit dimension to populate the inout_cd with the appropriate +PCORNet encounter type codes based on data in the observation_fact. + +The pcornet_mapping table contains columns that translate the PCORNet paths to +local paths. + +Example: +pcori_path,local_path +\PCORI\ENCOUNTER\ENC_TYPE\AV\,\i2b2\Visit Details\ENC_TYPE\AV\ +*/ +select * from pcornet_mapping pm where 1=0; + +whenever sqlerror continue; +drop table enc_type; +whenever sqlerror exit; + +/* explore encounter mapping: where do our encounters come from? */ +select count(*), count(distinct encounter_num), encounter_ide_source +from "&&i2b2_data_schema".encounter_mapping +group by encounter_ide_source +order by 1 desc +; + +select min(encounter_num), encounter_ide_source +from "&&i2b2_data_schema".encounter_mapping +group by encounter_ide_source +order by 1 +; + + +create table enc_type as +with +enc_type_codes as ( + select + pm.pcori_path, replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code, pm.local_path, cd.concept_cd + from pcornet_mapping pm + join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' + where pm.pcori_path like '\PCORI\ENCOUNTER\ENC_TYPE\%' + ) +-- TODO: Consider investigating why we have multiple encounter types for a single encounter +select obs.encounter_num, max(et.pcori_code) pcori_code +from "&&i2b2_data_schema".observation_fact obs +join enc_type_codes et on et.concept_cd = obs.concept_cd +group by obs.encounter_num +; + +create index enc_type_codes_encnum_idx on enc_type(encounter_num); + +update "&&i2b2_data_schema".visit_dimension vd +set inout_cd = coalesce(( + select et.pcori_code from enc_type et + where vd.encounter_num = et.encounter_num + ), 'UN'); + +-- Make sure the update went as expected +select case when count(*) > 0 then 1/0 else 1 end inout_cd_update_match from ( + select * from ( + select vd.inout_cd vd_code, et.pcori_code et_code + from "&&i2b2_data_schema".visit_dimension vd + left join enc_type et on et.encounter_num = vd.encounter_num + ) + where vd_code != vd_code and not (vd_code = 'UN' and vd_code is null) + ); diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index eaf01c0..1e7f54b 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -53,7 +53,7 @@ WHENEVER SQLERROR EXIT SQL.SQLCODE; select ethnicity_cd from "&&i2b2_data_schema".patient_dimension where 1=0; -- Make sure the inout_cd has been populated --- See update_inout_cd_vdim.sql +-- See heron_encounter_style.sql select case when qty = 0 then 1/0 else 1 end inout_cd_populated from ( select count(*) qty from "&&i2b2_data_schema".visit_dimension where inout_cd is not null ); diff --git a/Oracle/update_inout_cd_vdim.sql b/Oracle/update_inout_cd_vdim.sql deleted file mode 100644 index fce11e4..0000000 --- a/Oracle/update_inout_cd_vdim.sql +++ /dev/null @@ -1,49 +0,0 @@ -/* Update the visit dimension to populate the inout_cd with the appropriate -PCORNet encounter tyep codes based on data in the observation_fact. - -The pcornet_mapping table contains columns that translate the PCORNet paths to -local paths. - -Example: -pcori_path,local_path -\PCORI\ENCOUNTER\ENC_TYPE\AV\,\i2b2\Visit Details\ENC_TYPE\AV\ -*/ -select * from pcornet_mapping pm where 1=0; - -whenever sqlerror continue; -drop table enc_type; -whenever sqlerror exit; - -create table enc_type as -with -enc_type_codes as ( - select - pm.pcori_path, replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code, pm.local_path, cd.concept_cd - from pcornet_mapping pm - join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' - where pm.pcori_path like '\PCORI\ENCOUNTER\ENC_TYPE\%' - ) --- TODO: Consider investigating why we have multiple encounter types for a single encounter -select obs.encounter_num, max(et.pcori_code) pcori_code -from "&&i2b2_data_schema".observation_fact obs -join enc_type_codes et on et.concept_cd = obs.concept_cd -group by obs.encounter_num -; - -create index enc_type_codes_encnum_idx on enc_type(encounter_num); - -update "&&i2b2_data_schema".visit_dimension vd -set inout_cd = coalesce(( - select et.pcori_code from enc_type et - where vd.encounter_num = et.encounter_num - ), 'UN'); - --- Make sure the update went as expected -select case when count(*) > 0 then 1/0 else 1 end inout_cd_update_match from ( - select * from ( - select vd.inout_cd vd_code, et.pcori_code et_code - from "&&i2b2_data_schema".visit_dimension vd - left join enc_type et on et.encounter_num = vd.encounter_num - ) - where vd_code != vd_code and not (vd_code = 'UN' and vd_code is null) - ); From 3df5eede134c3dae25704fb49eb457d7ceffde75 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 11 Mar 2016 17:28:04 -0600 Subject: [PATCH 066/507] get discharge date from Discharge Disposition observation as well as UHC length_of_stay --- Oracle/heron_encounter_style.sql | 61 +++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 8d33eb3..9851898 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -18,6 +18,16 @@ create index diagnosis_encid on diagnosis (encounterid); create index procedures_patid on procedures (patid); create index procedures_encid on procedures (encounterid); create index vital_patid on vital (patid); + +actually, foreign key constraints make more sense... + +alter table encounter +add constraint fk_patid foreign key (patid) +references demographic (patid); + + +alter session set NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI'; + */ @@ -27,9 +37,10 @@ TODO: push this back into HERON ETL */ merge into "&&i2b2_data_schema".visit_dimension vd using ( + -- can we get it from UHC length_of_stay? select obs.encounter_num, obs.patient_num , obs.nval_num length_of_stay - , obs.end_date + , obs.end_date discharge_date from "&&i2b2_data_schema".observation_fact obs where obs.concept_cd = 'UHC|LOS:1' -- TODO: parameterize ) obs @@ -37,10 +48,34 @@ on ( obs.encounter_num = vd.encounter_num and obs.patient_num = vd.patient_num) when matched then update set vd.length_of_stay = obs.length_of_stay - , vd.end_date = obs.end_date + , vd.end_date = obs.discharge_date ; -- 194,328 rows merged. +merge into "&&i2b2_data_schema".visit_dimension vd +using ( + -- how about discharge disposition? + select obs.encounter_num, obs.patient_num + , max(obs.start_date) discharge_date + from "&&i2b2_data_schema".observation_fact obs + join ( + select distinct concept_cd + from "&&i2b2_data_schema".concept_dimension + -- TODO: map to PCORnet path + where concept_path like '\i2b2\Visit Details\Discharge Disposition Codes\%' + group by concept_cd + ) cd on cd.concept_cd = obs.concept_cd + group by encounter_num, patient_num +) obs +on ( obs.encounter_num = vd.encounter_num + and obs.patient_num = vd.patient_num) +when matched then update + set vd.length_of_stay = obs.discharge_date - vd.start_date + , vd.end_date = obs.discharge_date +; +-- 3,879,931 rows merged. + + merge into encounter pe using "&&i2b2_data_schema".visit_dimension vd @@ -61,13 +96,29 @@ with enc_agg as ( select count(*) enc_qty from encounter where enc_type in ('IP', 'IS') ) -select count(*), round(count(*) / enc_qty * 100, 1) pct -from encounter cross join enc_agg +select count(*), round(count(*) / enc_qty * 100, 1) pct, enc_type +from ( +select * +from encounter where enc_type in ('IP', 'IS') and discharge_date is null -group by enc_qty +) +cross join enc_agg +group by enc_qty, enc_type ; +/* Due to using hostpital accounts as encounters, +we have a long tail of very long encounters; hundreds of days. + +select count(*), los from ( +select round(discharge_date - admit_date) LOS +from encounter +where enc_type in ('IP', 'IS') +) group by los +order by 2 +; +*/ + /** TODO: chase down ~80% unknown enc_type */ From 164e0bf52944fad5557bdc0636402774dab2f09f Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 14 Mar 2016 10:54:17 -0500 Subject: [PATCH 067/507] Added failing test case to make sure at least some of the provider ids are populated. --- Oracle/cdm_transform_tests.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 6ab663e..5e2133d 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -161,3 +161,15 @@ calc as ( ) select case when sum(calc.tst) < 2 then 1/0 else 1 end pass from calc; +-- Make sure most provider ids in the visit dimension are not null/unknown/no information +select case when pct_not_null < 70 then 1/0 else 1 end some_providers_not_null from ( + with all_enc as ( + select count(*) qty from encounter + ), + not_null as ( + select count(*) qty from encounter + where providerid is not null and providerid not in ('NI', 'UN') + ) + select round((not_null.qty / all_enc.qty) * 100, 4) pct_not_null + from not_null cross join all_enc +); \ No newline at end of file From d435d777a24aa7fd9f92fde588e7eb48467eae82 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 14 Mar 2016 11:00:51 -0500 Subject: [PATCH 068/507] For the CDM encounter table, pull provider id from the visit dimension. - Remove TODOs and code commented out as a workaround. - Added test to the transform invocation script so it will fail early in the process if providerid doesn't exist. --- Oracle/PCORNetLoader_ora.sql | 5 +---- Oracle/run-i2p-transform.sh | 3 +++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 7eae7d5..7881a97 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -972,9 +972,6 @@ ORA-00904: "FACILITY_ID": invalid identifier 5) ORA-00904: "LOCATION_ZIP": invalid identifier - -6) -ORA-00904: "PROVIDERID": invalid identifier */ create or replace procedure PCORNetEncounter as @@ -990,7 +987,7 @@ select distinct v.patient_num, v.encounter_num, to_char(start_Date,'HH:MI'), end_Date, to_char(end_Date,'HH:MI'), - 'NI' providerid, --See TODO above + providerid, 'NI' location_zip, /* See TODO above */ (case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, 'NI' facility_id, /* See TODO above */ diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index eaf01c0..048d855 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -52,6 +52,9 @@ WHENEVER SQLERROR EXIT SQL.SQLCODE; -- See update_ethnicity_pdim.sql select ethnicity_cd from "&&i2b2_data_schema".patient_dimension where 1=0; +-- Make sure that the providerid column has been added to the visit dimension +select providerid from "&&i2b2_data_schema".visit_dimension where 1=0; + -- Make sure the inout_cd has been populated -- See update_inout_cd_vdim.sql select case when qty = 0 then 1/0 else 1 end inout_cd_populated from ( From 2facd5055ca2cb0b0c8b4127157be36f80ee051e Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 14 Mar 2016 14:09:52 -0500 Subject: [PATCH 069/507] Comment-out code that populates discharge date in the encounter table (let i2p-transform do this) - Add script to run both dimension update scripts. - Also comment out tests with TODOs to consider moving to our post-transform tests Thanks to Dan Connolly for design help. --- Oracle/heron_encounter_style.sql | 12 ++++++++++-- Oracle/update_to_cdm_dimensions.sh | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 Oracle/update_to_cdm_dimensions.sh diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 9851898..84be80e 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -76,13 +76,14 @@ when matched then update -- 3,879,931 rows merged. - +/* -- Manually update encounter (i2p-transform should fill in the column) merge into encounter pe using "&&i2b2_data_schema".visit_dimension vd on ( pe.encounterid = vd.encounter_num and vd.end_date is not null) when matched then update set pe.discharge_date = vd.end_date; -- 165,292 rows merged. +*/ /* Check that we have encounter.discharge_date for selected visit types. @@ -92,6 +93,8 @@ Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types." -- PCORnet CDM v3 */ + +/* TODO: Consider moving to cdm_transform_tests.sql with enc_agg as ( select count(*) enc_qty from encounter where enc_type in ('IP', 'IS') @@ -106,6 +109,7 @@ and discharge_date is null cross join enc_agg group by enc_qty, enc_type ; +*/ /* Due to using hostpital accounts as encounters, we have a long tail of very long encounters; hundreds of days. @@ -122,6 +126,7 @@ order by 2 /** TODO: chase down ~80% unknown enc_type */ +/* TODO: Consider moving to cdm_transform_tests.sql with enc_agg as ( select count(*) qty from encounter) select count(*) encounter_qty @@ -135,14 +140,17 @@ or exists ( select admit_date from procedures px where enc.patid = px.patid) group by enc_type, enc_agg.qty ; +*/ /* TODO: get encounter type from source system? */ +/* TODO: Consider moving to cdm_transform_tests.sql select count(*), sourcesystem_cd from nightherondata.visit_dimension group by sourcesystem_cd; */ /* TODO: get encounter type from place of service? */ +/* TODO: Consider moving to cdm_transform_tests.sql select obs.encounter_num, obs.patient_num, obs.start_date, obs.end_date , con.concept_cd, con.name_char place_of_service , obs.sourcesystem_cd @@ -152,7 +160,7 @@ join "&&i2b2_data_schema".visit_dimension v on v.encounter_num = obs.encounter_n where con.concept_path like '\i2b2\Visit Details\Place of Service (IDX)\%' and v.inout_cd = 'OT' ; - +*/ /* Update the visit dimension to populate the inout_cd with the appropriate PCORNet encounter type codes based on data in the observation_fact. diff --git a/Oracle/update_to_cdm_dimensions.sh b/Oracle/update_to_cdm_dimensions.sh new file mode 100644 index 0000000..ec6130d --- /dev/null +++ b/Oracle/update_to_cdm_dimensions.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# Expected environment variables (put there by Jenkins, etc.) +# export pcornet_cdm_user= +# export pcornet_cdm= + +sqlplus /nolog < Date: Tue, 8 Mar 2016 15:24:28 -0600 Subject: [PATCH 070/507] Added encounter type test from 5e654fc after it got lost in a merge. --- Oracle/cdm_transform_tests.sql | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 6ab663e..535429e 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -120,6 +120,19 @@ select case when pct_not_null < 99 then 1/0 else 1 end some_px_dates_not_null fr select case when count(*) = 0 then 1/0 else 1 end have_px_sources from ( select distinct px_source from procedures where px_source is not null ); + +-- Make sure we have some encounter types +select case when pct_known < 20 then 1/0 else 1 end some_known_enc_types from ( + with all_enc as ( + select count(*) qty from encounter + ), + known_enc as ( + select count(*) qty from encounter where enc_type is not null and enc_type != 'UN' + ) + select round((known_enc.qty / all_enc.qty) * 100, 4) pct_known + from known_enc cross join all_enc + ); + /* Test to make sure we have something about patient smoking tobacco use */ with snums as ( select smoking cat, count(smoking) qty from vital group by smoking From 06d647593f3806d7ecb7a6ffa488a9430333996a Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 14 Mar 2016 15:24:59 -0500 Subject: [PATCH 071/507] Move Dan's discharge date tests to cdm_transform_tests.sql. --- Oracle/cdm_transform_tests.sql | 53 ++++++++++++++++++++++++++++++ Oracle/heron_encounter_style.sql | 56 -------------------------------- 2 files changed, 53 insertions(+), 56 deletions(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 535429e..f47b10f 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -132,6 +132,23 @@ select case when pct_known < 20 then 1/0 else 1 end some_known_enc_types from ( select round((known_enc.qty / all_enc.qty) * 100, 4) pct_known from known_enc cross join all_enc ); + +/** TODO: chase down ~80% unknown enc_type */ +/* +with enc_agg as ( +select count(*) qty from encounter) +select count(*) encounter_qty + , round(count(*) / enc_agg.qty * 100, 1) pct + , enc_type +from encounter enc +cross join enc_agg +where exists ( + select admit_date from diagnosis dx where enc.patid = dx.patid) +or exists ( + select admit_date from procedures px where enc.patid = px.patid) +group by enc_type, enc_agg.qty +; +*/ /* Test to make sure we have something about patient smoking tobacco use */ with snums as ( @@ -174,3 +191,39 @@ calc as ( ) select case when sum(calc.tst) < 2 then 1/0 else 1 end pass from calc; + +/* Check that we have encounter.discharge_date for selected visit types. + +"Discharge date. Should be populated for all +Inpatient Hospital Stay (IP) and Non-Acute +Institutional Stay (IS) encounter types." + -- PCORnet CDM v3 +*/ + +select case when count(*) > 0 then 1/0 else 1 end pp_is_enc_have_discharge_date from ( + with enc_agg as ( + select count(*) enc_qty from encounter + where enc_type in ('IP', 'IS') + ) + select count(*), round(count(*) / enc_qty * 100, 1) pct, enc_type + from ( + select * + from encounter + where enc_type in ('IP', 'IS') + and discharge_date is null + ) + cross join enc_agg + group by enc_qty, enc_type + ); + +/* Due to using hostpital accounts as encounters, +we have a long tail of very long encounters; hundreds of days. + +select count(*), los from ( +select round(discharge_date - admit_date) LOS +from encounter +where enc_type in ('IP', 'IS') +) group by los +order by 2 +; +*/ diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 84be80e..710ec86 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -86,62 +86,6 @@ when matched then update set pe.discharge_date = vd.end_date; */ -/* Check that we have encounter.discharge_date for selected visit types. - -"Discharge date. Should be populated for all -Inpatient Hospital Stay (IP) and Non-Acute -Institutional Stay (IS) encounter types." - -- PCORnet CDM v3 -*/ - -/* TODO: Consider moving to cdm_transform_tests.sql -with enc_agg as ( - select count(*) enc_qty from encounter - where enc_type in ('IP', 'IS') - ) -select count(*), round(count(*) / enc_qty * 100, 1) pct, enc_type -from ( -select * -from encounter -where enc_type in ('IP', 'IS') -and discharge_date is null -) -cross join enc_agg -group by enc_qty, enc_type -; -*/ - -/* Due to using hostpital accounts as encounters, -we have a long tail of very long encounters; hundreds of days. - -select count(*), los from ( -select round(discharge_date - admit_date) LOS -from encounter -where enc_type in ('IP', 'IS') -) group by los -order by 2 -; -*/ - - - -/** TODO: chase down ~80% unknown enc_type */ -/* TODO: Consider moving to cdm_transform_tests.sql -with enc_agg as ( -select count(*) qty from encounter) -select count(*) encounter_qty - , round(count(*) / enc_agg.qty * 100, 1) pct - , enc_type -from encounter enc -cross join enc_agg -where exists ( - select admit_date from diagnosis dx where enc.patid = dx.patid) -or exists ( - select admit_date from procedures px where enc.patid = px.patid) -group by enc_type, enc_agg.qty -; -*/ - /* TODO: get encounter type from source system? */ /* TODO: Consider moving to cdm_transform_tests.sql select count(*), sourcesystem_cd From c28bd4933255ca7246f48196b0c19a247ce8499c Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 15 Mar 2016 09:51:40 -0500 Subject: [PATCH 072/507] curated mapping of IDX place of service name to ENC_TYPE --- Oracle/pcornet_mapping.csv | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 16263fc..51f6075 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -168,3 +168,29 @@ pcori_path,local_path \PCORI\ENCOUNTER\DISCHARGE_STATUS\HH\,\i2b2\Visit Details\Discharge Disposition Codes\252\ \PCORI\ENCOUNTER\DISCHARGE_STATUS\HS\,\i2b2\Visit Details\Discharge Disposition Codes\365\ \PCORI\ENCOUNTER\DRG_TYPE\02\,\i2b2\UHC\Diagnosis\Base MS DRG\ +"\PCORI\ENCOUNTER\ENC_TYPE\AV\","\i2b2\Visit Details\Place of Service (IDX)\22\" +"\PCORI\ENCOUNTER\ENC_TYPE\OA\","\i2b2\Visit Details\Place of Service (IDX)\03\" +"\PCORI\ENCOUNTER\ENC_TYPE\AV\","\i2b2\Visit Details\Place of Service (IDX)\26\" +"\PCORI\ENCOUNTER\ENC_TYPE\AV\","\i2b2\Visit Details\Place of Service (IDX)\11\" +"\PCORI\ENCOUNTER\ENC_TYPE\IP\","\i2b2\Visit Details\Place of Service (IDX)\21\" +"\PCORI\ENCOUNTER\ENC_TYPE\ED\","\i2b2\Visit Details\Place of Service (IDX)\23\" +"\PCORI\ENCOUNTER\ENC_TYPE\IS\","\i2b2\Visit Details\Place of Service (IDX)\13\" +"\PCORI\ENCOUNTER\ENC_TYPE\IS\","\i2b2\Visit Details\Place of Service (IDX)\65\" +"\PCORI\ENCOUNTER\ENC_TYPE\OA\","\i2b2\Visit Details\Place of Service (IDX)\34\" +"\PCORI\ENCOUNTER\ENC_TYPE\OT\","\i2b2\Visit Details\Place of Service (IDX)\25\" +"\PCORI\ENCOUNTER\ENC_TYPE\IP\","\i2b2\Visit Details\Place of Service (IDX)\51\" +"\PCORI\ENCOUNTER\ENC_TYPE\OA\","\i2b2\Visit Details\Place of Service (IDX)\55\" +"\PCORI\ENCOUNTER\ENC_TYPE\OA\","\i2b2\Visit Details\Place of Service (IDX)\12\" +"\PCORI\ENCOUNTER\ENC_TYPE\AV\","\i2b2\Visit Details\Place of Service (IDX)\20\" +"\PCORI\ENCOUNTER\ENC_TYPE\OA\","\i2b2\Visit Details\Place of Service (IDX)\52\" +"\PCORI\ENCOUNTER\ENC_TYPE\OA\","\i2b2\Visit Details\Place of Service (IDX)\72\" +"\PCORI\ENCOUNTER\ENC_TYPE\OT\","\i2b2\Visit Details\Place of Service (IDX)\9\" +"\PCORI\ENCOUNTER\ENC_TYPE\IS\","\i2b2\Visit Details\Place of Service (IDX)\32\" +"\PCORI\ENCOUNTER\ENC_TYPE\IS\","\i2b2\Visit Details\Place of Service (IDX)\33\" +"\PCORI\ENCOUNTER\ENC_TYPE\AV\","\i2b2\Visit Details\Place of Service (IDX)\24\" +"\PCORI\ENCOUNTER\ENC_TYPE\OT\","\i2b2\Visit Details\Place of Service (IDX)\99\" +"\PCORI\ENCOUNTER\ENC_TYPE\OA\","\i2b2\Visit Details\Place of Service (IDX)\81\" +"\PCORI\ENCOUNTER\ENC_TYPE\OA\","\i2b2\Visit Details\Place of Service (IDX)\56\" +"\PCORI\ENCOUNTER\ENC_TYPE\IS\","\i2b2\Visit Details\Place of Service (IDX)\31\" +"\PCORI\ENCOUNTER\ENC_TYPE\IS\","\i2b2\Visit Details\Place of Service (IDX)\54\" +"\PCORI\ENCOUNTER\ENC_TYPE\AV\","\i2b2\Visit Details\Place of Service (IDX)\19\" From 04c3e9e17b37a6198a1885b137f8e412511c2e76 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 15 Mar 2016 10:41:46 -0500 Subject: [PATCH 073/507] convert `enc_have_discharge_date` test to use test_cases table goal: run all test and do one 1/0 test at the end --- Oracle/cdm_transform_tests.sql | 44 ++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index fe143e9..9df3183 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -1,3 +1,22 @@ +whenever sqlerror continue +drop table test_cases; +whenever sqlerror exit; +create table test_cases ( + pass integer, + query_name varchar2(160), + value varchar2(160), + freq number, + pct number, + description varchar2(2000) +) +; +comment on column test_cases.pass is '1 for pass, 0 for fail'; +comment on column test_cases.query_name is 'a la DRN OS SAS script query name'; +comment on column test_cases.value is 'relevant nominal or scalar value'; +comment on column test_cases.freq is 'frequency; i.e. count our quanitity'; +comment on column test_cases.pct is 'percent of all observations'; + + /* Test to make sure we got about the same number of patients in the CDM diagnoses that we do in i2b2. */ @@ -204,29 +223,35 @@ select case when pct_not_null < 70 then 1/0 else 1 end some_providers_not_null f from not_null cross join all_enc ); -/* Check that we have encounter.discharge_date for selected visit types. +insert into test_cases (query_name, description, pass, freq, pct, value) +select 'enc_have_discharge_date' query_name + , ' "Discharge date. Should be populated for all Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types." -- PCORnet CDM v3 -*/ - -select case when count(*) > 0 then 1/0 else 1 end pp_is_enc_have_discharge_date from ( +' description + , case when t.freq = 0 then 1 else 0 end pass + , t.* +from ( with enc_agg as ( select count(*) enc_qty from encounter where enc_type in ('IP', 'IS') ) - select count(*), round(count(*) / enc_qty * 100, 1) pct, enc_type + select count(*) freq + , round(count(*) / enc_qty * 100, 1) pct + , enc_type value from ( select * from encounter where enc_type in ('IP', 'IS') and discharge_date is null - ) + ) problems cross join enc_agg group by enc_qty, enc_type - ); + ) t + ; /* Due to using hostpital accounts as encounters, we have a long tail of very long encounters; hundreds of days. @@ -239,3 +264,8 @@ where enc_type in ('IP', 'IS') order by 2 ; */ + +select case when count(*) > 0 then 1/0 else 1 end all_test_cases_pass from ( +select * from test_cases where pass = 0 +); + From b8119da7bc27361bf7084c5697c1fdd958300ad7 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 15 Mar 2016 10:57:48 -0500 Subject: [PATCH 074/507] convert some_providers_not_null test to use test_cases table --- Oracle/cdm_transform_tests.sql | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 9df3183..6469056 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -210,18 +210,29 @@ calc as ( ) select case when sum(calc.tst) < 2 then 1/0 else 1 end pass from calc; --- Make sure most provider ids in the visit dimension are not null/unknown/no information -select case when pct_not_null < 70 then 1/0 else 1 end some_providers_not_null from ( + +insert into test_cases (query_name, description, pass, freq, pct, value) +select 'some_providers_not_null' query_name + , 'Make sure most provider ids in the visit dimension are + not null/unknown/no information' description + , case when value = 'OK' and pct > 70 then 1 + when value = 'UN' and pct < 30 then 1 + else 0 end pass + , t.* +from ( with all_enc as ( - select count(*) qty from encounter - ), - not_null as ( - select count(*) qty from encounter - where providerid is not null and providerid not in ('NI', 'UN') - ) - select round((not_null.qty / all_enc.qty) * 100, 4) pct_not_null - from not_null cross join all_enc -); + select count(*) qty from encounter), + provider_ok as ( + select case when providerid is not null + and providerid not in ('NI', 'UN') then 'OK' + else 'UN' + end value + from encounter) + select count(*) freq, round(count(*) / all_enc.qty * 100, 4) pct, value + from provider_ok cross join all_enc + group by value, all_enc.qty +) t +; insert into test_cases (query_name, description, pass, freq, pct, value) From 66db1e45938682129220e9863d187879d0fb39b4 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 15 Mar 2016 14:13:29 -0500 Subject: [PATCH 075/507] refine test_results to match ENC_L3_ENCTYPE_ADATE_YM etc. - un-comment unknown enc_type test; rewrite to match ENC_L3_ENCTYPE_ADATE_YM - add columns: obs, by_time, by_value2, ... - rename freq, pct to record_n, record_pct - rewrite some_providers_not_null as ENC_L3_N - rewrite enc_have_discharge_date as ENC_L3_ENCTYPE_DDATE_YM --- Oracle/cdm_transform_tests.sql | 175 +++++++++++++++++++++------------ 1 file changed, 112 insertions(+), 63 deletions(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 6469056..862371f 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -2,19 +2,27 @@ whenever sqlerror continue drop table test_cases; whenever sqlerror exit; create table test_cases ( - pass integer, - query_name varchar2(160), - value varchar2(160), - freq number, - pct number, + query_name varchar2(160) not null, + obs integer not null, + by_time integer, + by_value1 varchar2(160), + by_value2 varchar2(160), + record_n integer, + record_pct number, + distinct_patid_n integer, + pass integer not null, description varchar2(2000) ) ; -comment on column test_cases.pass is '1 for pass, 0 for fail'; comment on column test_cases.query_name is 'a la DRN OS SAS script query name'; -comment on column test_cases.value is 'relevant nominal or scalar value'; -comment on column test_cases.freq is 'frequency; i.e. count our quanitity'; -comment on column test_cases.pct is 'percent of all observations'; +comment on column test_cases.obs is 'Obs / line number'; +comment on column test_cases.by_time is 'breakdown by year or year/month'; +comment on column test_cases.by_value1 is 'relevant nominal value'; +comment on column test_cases.by_value2 is '2nd level breakdown'; +comment on column test_cases.record_n is 'count our quanitity'; +comment on column test_cases.record_pct is 'percent of all observations'; +comment on column test_cases.distinct_patid_n is 'patient count'; +comment on column test_cases.pass is '1 for pass, 0 for fail'; /* Test to make sure we got about the same number of patients in the CDM @@ -152,22 +160,43 @@ select case when pct_known < 20 then 1/0 else 1 end some_known_enc_types from ( from known_enc cross join all_enc ); -/** TODO: chase down ~80% unknown enc_type */ -/* -with enc_agg as ( -select count(*) qty from encounter) -select count(*) encounter_qty - , round(count(*) / enc_agg.qty * 100, 1) pct - , enc_type -from encounter enc -cross join enc_agg -where exists ( - select admit_date from diagnosis dx where enc.patid = dx.patid) -or exists ( - select admit_date from procedures px where enc.patid = px.patid) -group by enc_type, enc_agg.qty + +insert into test_cases (query_name, description, pass + , obs, record_n, record_pct, by_time, by_value1) +select 'ENC_L3_ENCTYPE_ADATE_YM' query_name + , 'TODO: chase down ~80% unknown enc_type' description + , case when enc_type in ('UN', 'OT') and pct < 25 then 1 + when enc_type not in ('UN', 'OT') and freq > 1 then 1 + else 0 end pass + , rownum obs + , t.* +from ( +with enc as ( + select extract (year from admit_date) * 100 + + extract (month from admit_date) admit_ym + , enc_type + from encounter +), enc_agg as ( + select count(*) qty from encounter +), by_time as ( + select count(*) qty, admit_ym + from enc group by admit_ym +), by_val as ( + select count(*) freq + , round(count(*) / enc_agg.qty * 100, 1) pct + , admit_ym + , enc_type + from enc cross join enc_agg + group by admit_ym, enc_type, enc_agg.qty + order by admit_ym, enc_type) +select by_val.freq + , round(by_val.freq / by_time.qty * 100, 4) pct + , by_val.admit_ym + , by_val.enc_type +from by_time join by_val on by_time.admit_ym = by_val.admit_ym +) t ; -*/ + /* Test to make sure we have something about patient smoking tobacco use */ with snums as ( @@ -211,58 +240,78 @@ calc as ( select case when sum(calc.tst) < 2 then 1/0 else 1 end pass from calc; -insert into test_cases (query_name, description, pass, freq, pct, value) -select 'some_providers_not_null' query_name - , 'Make sure most provider ids in the visit dimension are - not null/unknown/no information' description - , case when value = 'OK' and pct > 70 then 1 - when value = 'UN' and pct < 30 then 1 +insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) +select 'ENC_L3_N' query_name + , 'providerid: many distinct' description + , case when distinct_n / all_n * 100 >= 2 then 1 else 0 end pass - , t.* + , rownum obs + , t.tag, t.distinct_n, round(t.distinct_n / t.all_n * 100, 4) from ( - with all_enc as ( - select count(*) qty from encounter), - provider_ok as ( - select case when providerid is not null - and providerid not in ('NI', 'UN') then 'OK' - else 'UN' - end value - from encounter) - select count(*) freq, round(count(*) / all_enc.qty * 100, 4) pct, value - from provider_ok cross join all_enc - group by value, all_enc.qty +with enc as ( +select encounterid + , case when encounterid is null then 1 else null end encounterid_null + , patid + , case when patid is null then 1 else null end patid_null + , providerid + , case when providerid is null then 1 else null end providerid_null +from encounter +) +select 'encounterid' tag + , count(encounterid) all_n, count(distinct encounterid) distinct_n, count(encounterid_null) null_n +from enc +union all +select 'patid' tag + , count(patid), count(distinct patid), count(patid_null) +from enc +union all +select 'providerid' tag + , count(providerid), count(distinct providerid), count(providerid_null) +from enc ) t ; -insert into test_cases (query_name, description, pass, freq, pct, value) -select 'enc_have_discharge_date' query_name +insert into test_cases (query_name, description, pass, obs, by_value1, by_time, record_n, record_pct, distinct_patid_n) +select 'ENC_L3_ENCTYPE_DDATE_YM' query_name , ' "Discharge date. Should be populated for all Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types." -- PCORnet CDM v3 ' description - , case when t.freq = 0 then 1 else 0 end pass - , t.* + , case when enc_type not in ('IP', 'IS') then 1 + when discharge_date is not null then 1 + when record_pct < 10 then 1 + else 0 + end pass + , q.* from ( - with enc_agg as ( - select count(*) enc_qty from encounter - where enc_type in ('IP', 'IS') - ) - select count(*) freq - , round(count(*) / enc_qty * 100, 1) pct - , enc_type value - from ( - select * - from encounter - where enc_type in ('IP', 'IS') - and discharge_date is null - ) problems - cross join enc_agg - group by enc_qty, enc_type - ) t - ; + select rownum obs, q.* from ( + with enc_agg as ( + select count(*) tot from encounter + ), + enc as ( + select extract(year from discharge_date) * 100 + + extract(month from discharge_date) discharge_date + , encounterid + , patid + , enc_type + , case when discharge_date is null + or discharge_date = 'NI' + then 1 else 0 end discharge_date_null + from encounter ) + select enc_type + , discharge_date + , count(*) record_n + , round(count(*) / enc_agg.tot * 100, 4) record_pct + , count(distinct patid) distinct_patid_n + from enc cross join enc_agg + group by discharge_date, enc_type, enc_agg.tot + order by 3 desc) q + ) q +where q.obs <= 100; + /* Due to using hostpital accounts as encounters, we have a long tail of very long encounters; hundreds of days. From 0cc5ea6c4429f1ce1eb8a64b07b52f70cad483cc Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 15 Mar 2016 14:22:22 -0500 Subject: [PATCH 076/507] convert inout_cd update to use `merge into ...` move manual update of `encounter` after inout_cd merge --- Oracle/heron_encounter_style.sql | 69 ++++++++++++++++---------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 710ec86..04372c0 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -76,16 +76,6 @@ when matched then update -- 3,879,931 rows merged. -/* -- Manually update encounter (i2p-transform should fill in the column) -merge into encounter pe -using "&&i2b2_data_schema".visit_dimension vd - on ( pe.encounterid = vd.encounter_num - and vd.end_date is not null) -when matched then update set pe.discharge_date = vd.end_date; --- 165,292 rows merged. -*/ - - /* TODO: get encounter type from source system? */ /* TODO: Consider moving to cdm_transform_tests.sql select count(*), sourcesystem_cd @@ -136,36 +126,47 @@ order by 1 ; -create table enc_type as -with -enc_type_codes as ( - select - pm.pcori_path, replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code, pm.local_path, cd.concept_cd +merge into "&&i2b2_data_schema".visit_dimension vd +using ( + with + enc_type_codes as ( + select pm.pcori_path + , replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code + , pm.local_path + , cd.concept_cd from pcornet_mapping pm join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' where pm.pcori_path like '\PCORI\ENCOUNTER\ENC_TYPE\%' ) --- TODO: Consider investigating why we have multiple encounter types for a single encounter +-- Note: our patient-day style encounters etc. may result in +-- multiple encounter types for a single encounter. select obs.encounter_num, max(et.pcori_code) pcori_code from "&&i2b2_data_schema".observation_fact obs join enc_type_codes et on et.concept_cd = obs.concept_cd group by obs.encounter_num -; +) et on ( vd.encounter_num = et.encounter_num ) +when matched then update +set inout_cd = et.pcori_code; +-- 11,085,911 rows merged. + -create index enc_type_codes_encnum_idx on enc_type(encounter_num); - -update "&&i2b2_data_schema".visit_dimension vd -set inout_cd = coalesce(( - select et.pcori_code from enc_type et - where vd.encounter_num = et.encounter_num - ), 'UN'); - --- Make sure the update went as expected -select case when count(*) > 0 then 1/0 else 1 end inout_cd_update_match from ( - select * from ( - select vd.inout_cd vd_code, et.pcori_code et_code - from "&&i2b2_data_schema".visit_dimension vd - left join enc_type et on et.encounter_num = vd.encounter_num - ) - where vd_code != vd_code and not (vd_code = 'UN' and vd_code is null) - ); +/* -- Manually update encounter (i2p-transform should fill in the column) + +select count(*) from encounter; +-- 14,665,341 + +merge into encounter pe +using "&&i2b2_data_schema".visit_dimension vd + on ( pe.encounterid = vd.encounter_num + and vd.end_date is not null) +when matched then update set pe.discharge_date = vd.end_date; +-- 165,292 rows merged. + +merge into encounter pe +using "&&i2b2_data_schema".visit_dimension vd + on ( pe.encounterid = vd.encounter_num + and vd.inout_cd is not null) +when matched then update + set pe.enc_type = vd.inout_cd; + -- 14,665,341 rows merged. +*/ From 394ad6d5f1399b727e8ffdca2e9c17cfbb70e226 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 15 Mar 2016 14:48:33 -0500 Subject: [PATCH 077/507] add ENC_L3_DISDISP test: too many NI in discharge_disposition with Nathan --- Oracle/cdm_transform_tests.sql | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 862371f..cb8b8e5 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -313,6 +313,28 @@ from ( where q.obs <= 100; +insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) +select 'ENC_L3_DISDISP' query_name + , 'too many NI' description + , case when discharge_disposition = 'NI' and record_pct < 40 then 1 + when discharge_disposition is null and record_pct < 5 then 1 + when discharge_disposition != 'NI' then 1 + else 0 + end pass + , rownum obs + , q.* +from ( +with enc_agg as ( + select count(*) tot from encounter) +select discharge_disposition + , count(*) record_n + , round(count(*) / enc_agg.tot * 100, 4) record_pct +from encounter cross join enc_agg +group by discharge_disposition, enc_agg.tot +order by 1 +) q +; + /* Due to using hostpital accounts as encounters, we have a long tail of very long encounters; hundreds of days. From e85bc28cca793c41177d014fec40aff5f509c5a3 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 15 Mar 2016 15:11:15 -0500 Subject: [PATCH 078/507] Simple pass/fail test to make sure not all discharge dispositions are "NI" --- Oracle/cdm_transform_tests.sql | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index fe143e9..e047089 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -239,3 +239,22 @@ where enc_type in ('IP', 'IS') order by 2 ; */ + +/* Make sure we have some discharge disposition data. + +DISCHARGE_DISPOSITION: Vital status at discharge. Should be populated for +Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter +types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) +encounter types. Should be missing for ambulatory visit (AV or OA) + +from: 2015-07-29-PCORnet-Common-Data-Model-v3dot0-RELEASE.pdf + +TODO: Make this test more robust - at the moment, it just makes sure not all are +NI. +*/ +select case when qty = 0 then 1/0 else 1 end some_discharge_dispositions from ( + select count(*) qty + from encounter e + where e.discharge_disposition is not null + and e.discharge_disposition != 'NI' + ); From 06228c7b7c957a37165a1be29a8d799cd99935ab Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 15 Mar 2016 15:14:35 -0500 Subject: [PATCH 079/507] Populate discharge disposition in the encounter table. - Uncomment discharge disposition references that were temporarily commented out to get the code to run. - Add code to populate the i2b2 visit dimension with discharge disposition. - Prioritize if there are multiple values found per encounter. --- Oracle/PCORNetLoader_ora.sql | 5 +-- Oracle/heron_encounter_style.sql | 65 ++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 7881a97..19ed5d3 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -964,9 +964,6 @@ Commented out that and what it's used for (ADMITTING_SOURCE) 2) ORA-00904: "DISCHARGE_STATUS": invalid identifier -3) -ORA-00904: "DISCHARGE_DISPOSITION": invalid identifier - 4) ORA-00904: "FACILITY_ID": invalid identifier @@ -991,7 +988,7 @@ select distinct v.patient_num, v.encounter_num, 'NI' location_zip, /* See TODO above */ (case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, 'NI' facility_id, /* See TODO above */ - 'NI' discharge_disposition /*CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END*/ , + CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END, 'NI' discharge_status /* See TODO above CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END*/ , drg.drg, drg_type, 'NI' admitting_source /* see TODO above CASE WHEN _source IS NULL THEN 'NI' ELSE admitting_source END*/ from i2b2visit v inner join demographic d on v.patient_num=d.patid diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 710ec86..4e37689 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -169,3 +169,68 @@ select case when count(*) > 0 then 1/0 else 1 end inout_cd_update_match from ( ) where vd_code != vd_code and not (vd_code = 'UN' and vd_code is null) ); + + +/* Discharge disposition +Valid values are: + +A=Discharged alive +E=Expired +NI=No information +UN=Unknown +OT=Other + +So, if there are multiple discharge dispositions per encounter (as we've observed +in test data at least) prioritize and pick one. +*/ +whenever sqlerror continue; +alter table "&&i2b2_data_schema".visit_dimension add ( + discharge_disposition VARCHAR2(4 BYTE) + ); +drop table enc_disp_facts; +drop table discharge_disp; +whenever sqlerror exit; + +-- Two steps (tables) due to Oracle performance/picking bad plans when it's all one query +create table enc_disp_facts as +with +enc_type_codes as ( + select + pm.pcori_path, replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code, pm.local_path, cd.concept_cd + from pcornet_mapping pm + --join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' + join nightherondata.concept_dimension cd on cd.concept_path like pm.local_path || '%' + where pm.pcori_path like '\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\%' + ) +select obs.encounter_num, et.pcori_code +from "&&i2b2_data_schema".observation_fact obs +join enc_type_codes et on et.concept_cd = obs.concept_cd; + +create table discharge_disp as +with +ranks as ( + select + encounter_num, pcori_code, + -- Prioritize disposition + decode(pcori_code, 'E', 0, 'A', 1, 'OT', 2, 'UN', 3, 'NI', 4, 99) pc_rank + from enc_disp_facts + ), +priority_rank as ( + -- TODO: Consider figuring out why we have multiple discharge dispositions per encounter. + select min(pc_rank) pc_rank, encounter_num + from ranks + group by encounter_num + ) +select distinct + ranks.encounter_num, coalesce(ranks.pcori_code, 'NI') pcori_code +from priority_rank pr +join ranks on pr.encounter_num = ranks.encounter_num and pr.pc_rank = ranks.pc_rank +; + +create index discharge_disp_encnum_idx on discharge_disp(encounter_num); + +update "&&i2b2_data_schema".visit_dimension vd +set discharge_disposition = coalesce(( + select dd.pcori_code from discharge_disp dd + where vd.encounter_num = dd.encounter_num + ), 'UN'); From 0d371a7258db44890f997205539650cce531b3e3 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 15 Mar 2016 17:25:56 -0500 Subject: [PATCH 080/507] Test report entry for discharge status. --- Oracle/cdm_transform_tests.sql | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index cb8b8e5..4f1f580 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -335,6 +335,30 @@ order by 1 ) q ; + +insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) +select 'ENC_L3_DISSTAT' query_name + , 'too many NI' description + , case when discharge_status = 'NI' and record_pct < 40 then 1 + when discharge_status is null and record_pct < 5 then 1 + when discharge_status != 'NI' then 1 + else 0 + end pass + , rownum obs + , q.* +from ( +with enc_agg as ( + select count(*) tot from encounter) +select discharge_status + , count(*) record_n + , round(count(*) / enc_agg.tot * 100, 4) record_pct +from encounter cross join enc_agg +group by discharge_status, enc_agg.tot +order by 1 +) q +; + + /* Due to using hostpital accounts as encounters, we have a long tail of very long encounters; hundreds of days. From a7b087e1646b445cff040e4875b76ed89bb0c2cc Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 15 Mar 2016 17:28:24 -0500 Subject: [PATCH 081/507] Populate discharge status in the encounter table. - Remove hard-coded "NI", replaced with commented-out case statement. - Populate i2b2 visit_dimension with discharge status - prioritize when there are multiple status entries per encounter. --- Oracle/PCORNetLoader_ora.sql | 5 +-- Oracle/heron_encounter_style.sql | 57 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 7881a97..ccd6251 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -961,9 +961,6 @@ Error(16,266): PL/SQL: ORA-00911: invalid character Commented out that and what it's used for (ADMITTING_SOURCE) -2) -ORA-00904: "DISCHARGE_STATUS": invalid identifier - 3) ORA-00904: "DISCHARGE_DISPOSITION": invalid identifier @@ -992,7 +989,7 @@ select distinct v.patient_num, v.encounter_num, (case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, 'NI' facility_id, /* See TODO above */ 'NI' discharge_disposition /*CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END*/ , - 'NI' discharge_status /* See TODO above CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END*/ , + CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END, drg.drg, drg_type, 'NI' admitting_source /* see TODO above CASE WHEN _source IS NULL THEN 'NI' ELSE admitting_source END*/ from i2b2visit v inner join demographic d on v.patient_num=d.patid left outer join diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 710ec86..4e4a42c 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -169,3 +169,60 @@ select case when count(*) > 0 then 1/0 else 1 end inout_cd_update_match from ( ) where vd_code != vd_code and not (vd_code = 'UN' and vd_code is null) ); + + +/* Discharge status +There may be multiple per encounter - rank and pick one. +*/ +whenever sqlerror continue; +alter table "&&i2b2_data_schema".visit_dimension add ( + discharge_status VARCHAR2(4 BYTE) + ); +drop table enc_status_facts; +drop table discharge_status; +whenever sqlerror exit; + +-- Two steps (tables) due to Oracle performance/picking bad plans when it's all one query +create table enc_status_facts as +with +enc_status_codes as ( + select + pm.pcori_path, replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code, pm.local_path, cd.concept_cd + from pcornet_mapping pm + join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' + where pm.pcori_path like '\PCORI\ENCOUNTER\DISCHARGE_STATUS\%' + ) +select obs.encounter_num, et.pcori_code +from "&&i2b2_data_schema".observation_fact obs +join enc_status_codes et on et.concept_cd = obs.concept_cd; + +create table discharge_status as +with +ranks as ( + select + encounter_num, pcori_code, + -- Prioritize status: First EX, Middle: , Last OT, UN, NI + decode(pcori_code, 'EX',0,'AF',1,'AL',2,'AM',3,'AW',4,'HH',5,'HO',6,'HS',7, + 'IP',8,'NH',9,'RH',10,'RS',11,'SH',12,'SN',12,'OT',96, + 'UN',97,'NI',98,99) pc_rank + from enc_status_facts + ), +priority_rank as ( + -- TODO: Consider figuring out why we have multiple discharge status values per encounter. + select min(pc_rank) pc_rank, encounter_num + from ranks + group by encounter_num + ) +select distinct + ranks.encounter_num, coalesce(ranks.pcori_code, 'NI') pcori_code +from priority_rank pr +join ranks on pr.encounter_num = ranks.encounter_num and pr.pc_rank = ranks.pc_rank +; + +create index discharge_status_encnum_idx on discharge_status(encounter_num); + +update "&&i2b2_data_schema".visit_dimension vd +set discharge_status = coalesce(( + select ds.pcori_code from discharge_status ds + where vd.encounter_num = ds.encounter_num + ), 'UN'); From b7d4f994c211a6e46e6b7063cb18f97959079035 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 15 Mar 2016 17:30:03 -0500 Subject: [PATCH 082/507] Oops - remove leftover hard-coded reference to HERON schema. --- Oracle/heron_encounter_style.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 4e37689..f4782b6 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -198,8 +198,7 @@ enc_type_codes as ( select pm.pcori_path, replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code, pm.local_path, cd.concept_cd from pcornet_mapping pm - --join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' - join nightherondata.concept_dimension cd on cd.concept_path like pm.local_path || '%' + join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' where pm.pcori_path like '\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\%' ) select obs.encounter_num, et.pcori_code From 39c752564032061672c8c0a128518e5cd4ce976d Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 16 Mar 2016 08:52:26 -0500 Subject: [PATCH 083/507] adjust threshold for ENC_L3_N test: 10K providers is plenty --- Oracle/cdm_transform_tests.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 862371f..f134b9a 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -243,7 +243,8 @@ select case when sum(calc.tst) < 2 then 1/0 else 1 end pass from calc; insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) select 'ENC_L3_N' query_name , 'providerid: many distinct' description - , case when distinct_n / all_n * 100 >= 2 then 1 + , case when distinct_n / all_n * 100 >= 0.01 + and null_n / all_n * 100 < 40 then 1 else 0 end pass , rownum obs , t.tag, t.distinct_n, round(t.distinct_n / t.all_n * 100, 4) From 0260899247f4baca8add2138ffc8904e572d3415 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 16 Mar 2016 09:14:41 -0500 Subject: [PATCH 084/507] First pass at populating encounter DRGs - TODO: Replace simple 1/0 test with a test report entry. --- Oracle/cdm_transform_tests.sql | 11 +++++++ Oracle/pcornet_mapping.csv | 2 +- Oracle/pcornet_mapping.sql | 54 ++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index cb8b8e5..b15d4be 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -240,6 +240,17 @@ calc as ( select case when sum(calc.tst) < 2 then 1/0 else 1 end pass from calc; +/* Simple test to make sure we have some MS-DRGs. +TODO: Rewrite this as the new-style test (insert into test_cases): ENC_L3_DRG +*/ +select case when unique_drgs < 10 then 1/0 else 1 end many_different_drgs from ( + select count(distinct drg) unique_drgs + from ( + select drg from encounter where drg is not null + ) + ); + + insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) select 'ENC_L3_N' query_name , 'providerid: many distinct' description diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 51f6075..d9de85e 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -167,7 +167,7 @@ pcori_path,local_path \PCORI\ENCOUNTER\DISCHARGE_STATUS\IP\,\i2b2\Visit Details\Discharge Disposition Codes\263\ \PCORI\ENCOUNTER\DISCHARGE_STATUS\HH\,\i2b2\Visit Details\Discharge Disposition Codes\252\ \PCORI\ENCOUNTER\DISCHARGE_STATUS\HS\,\i2b2\Visit Details\Discharge Disposition Codes\365\ -\PCORI\ENCOUNTER\DRG_TYPE\02\,\i2b2\UHC\Diagnosis\Base MS DRG\ +\PCORI\ENCOUNTER\DRG\02\,\i2b2\UHC\Diagnosis\UHC MS-DRG\ "\PCORI\ENCOUNTER\ENC_TYPE\AV\","\i2b2\Visit Details\Place of Service (IDX)\22\" "\PCORI\ENCOUNTER\ENC_TYPE\OA\","\i2b2\Visit Details\Place of Service (IDX)\03\" "\PCORI\ENCOUNTER\ENC_TYPE\AV\","\i2b2\Visit Details\Place of Service (IDX)\26\" diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 625ee9c..9e1d25a 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -213,3 +213,57 @@ from "&&i2b2_meta_schema"."&&terms_table" ht where c_fullname like '\i2b2\Procedures\PRC\Metathesaurus CPT Hierarchical Terms\%' order by c_hlevel ; + +/* MS-DRGs +*/ +delete +from "&&i2b2_meta_schema".PCORNET_ENC +where c_fullname like '\PCORI\ENCOUNTER\DRG\02\%' +; + +insert into "&&i2b2_meta_schema".PCORNET_ENC +with drg_path_map as ( + select local_path, local_path || '%' like_local_path, pcori_path, pcori_path || '%' like_pcori_path + from pcornet_mapping where pcori_path like '\PCORI\ENCOUNTER\DRG\02\%' + ) +select + ht.c_hlevel, + replace(ht.c_fullname, pm.local_path, pm.pcori_path) c_fullname, + ht.c_name, ht.c_synonym_cd, ht.c_visualattributes, + ht.c_totalnum, ht.c_basecode, ht.c_metadataxml, ht.c_facttablecolumn, ht.c_tablename, + ht.c_columnname, ht.c_columndatatype, ht.c_operator, ht.c_dimcode, ht.c_comment, + ht.c_tooltip, ht.m_applied_path, ht.update_date, ht.download_date, ht.import_date, + ht.sourcesystem_cd, ht.valuetype_cd, ht.m_exclusion_cd, ht.c_path, ht.c_symbol, + --TODO: Consider derviving the appropriate replacement strings. + replace(ht.c_basecode, 'UHC|UHCMSDRG:', 'MSDRG:') pcori_basecode +from + "&&i2b2_meta_schema"."&&terms_table" ht +cross join drg_path_map pm +where c_fullname like pm.like_local_path +--order by c_hlevel +; + +/* The following is good for eyeballing the DRG alignment - the names seem to +match exactly between UHC and the PCORNet ontology. +*/ +/* +with drg_path_map as ( + select local_path, local_path || '%' like_local_path, pcori_path, pcori_path || '%' like_pcori_path + from pcornet_mapping where pcori_path like '\PCORI\ENCOUNTER\DRG\02\%' + ), +pcornet_drgs as ( + select replace(pe.pcori_basecode, 'MSDRG:', '') drg_code, pe.* + from "&&i2b2_meta_schema".pcornet_enc pe + cross join drg_path_map dm + where pe.c_fullname like dm.like_pcori_path + ), +local_drgs as ( + select + replace(ht.c_basecode, 'UHC|UHCMSDRG:', '') drg_code, ht.* + from "&&i2b2_meta_schema"."&&terms_table" ht + cross join drg_path_map dm + where ht.c_fullname like dm.like_local_path + ) +select pd.drg_code, pd.c_name, ld.c_name from pcornet_drgs pd +join local_drgs ld on ld.drg_code = pd.drg_code; +*/ From d17ebd6767b766471df610f68644d43ec3ad4cb8 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 16 Mar 2016 09:23:54 -0500 Subject: [PATCH 085/507] migrate smoking_count_ok test to VIT_L3_SMOKING in test_cases table --- Oracle/cdm_transform_tests.sql | 50 ++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index f134b9a..390e288 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -198,20 +198,46 @@ from by_time join by_val on by_time.admit_ym = by_val.admit_ym ; -/* Test to make sure we have something about patient smoking tobacco use */ -with snums as ( - select smoking cat, count(smoking) qty from vital group by smoking + +select 'VIT_L3_SMOKING' query_name + , 'make sure we have something about patient smoking tobacco use' description + , case + when smoking = 'NI' and record_pct < 95 then 1 + when smoking != 'NI' and distinct_patid_n > 2 then 1 + else 0 + end pass + , rownum obs + , q.* +from ( +with smoking_terms as ( +select distinct pcori_basecode +from pcornet_vital +where c_fullname like '\PCORI\VITAL\TOBACCO\SMOKING\%' ), -tot as ( - select sum(qty) as cnt from snums +vit as ( +select patid + , smoking + , case when smoking = 'NI' then null + when exists ( + select pcori_basecode + from smoking_terms + where smoking_terms.pcori_basecode = vital.smoking) + then null + else 1 + end value_outside +from vital ), -calc as ( - select snums.cat, (snums.qty/tot.cnt*100) pct, - case when (snums.qty/tot.cnt*100) > 1 then 1 else 0 end tst - from snums, tot - where snums.cat!='NI' -) -select case when sum(calc.tst) < 3 then 1/0 else 1 end smoking_count_ok from calc; +agg as ( + select count(*) tot from vital) +select smoking, value_outside + , count(*) + , round(count(*) / tot * 100, 4) record_pct + , count(distinct patid) distinct_patid_n +from vit cross join agg +group by smoking, value_outside, tot +order by smoking +) q +; /* Test to make sure we have something about patient general tobacco use */ From 6fd35c4c7fa1a9a465d945eb838b88fad663aae8 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 16 Mar 2016 09:46:22 -0500 Subject: [PATCH 086/507] Zero-pad MS-DRGs as specified by the CDM v3 specification. --- Oracle/pcornet_mapping.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 9e1d25a..4c7df4b 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -234,8 +234,9 @@ select ht.c_columnname, ht.c_columndatatype, ht.c_operator, ht.c_dimcode, ht.c_comment, ht.c_tooltip, ht.m_applied_path, ht.update_date, ht.download_date, ht.import_date, ht.sourcesystem_cd, ht.valuetype_cd, ht.m_exclusion_cd, ht.c_path, ht.c_symbol, - --TODO: Consider derviving the appropriate replacement strings. - replace(ht.c_basecode, 'UHC|UHCMSDRG:', 'MSDRG:') pcori_basecode + case when ht.c_basecode is not null + then 'MSDRG:' || lpad(substr(ht.c_basecode, instr(ht.c_basecode, ':') + 1), 3, '0') + end pcori_basecode from "&&i2b2_meta_schema"."&&terms_table" ht cross join drg_path_map pm From 703f21e8addd38d22a4cd08d9bb5fcee8c5ee001 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 16 Mar 2016 10:40:09 -0500 Subject: [PATCH 087/507] migrate many_different_drgs test to match ENC_L3_DRG --- Oracle/cdm_transform_tests.sql | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index b15d4be..d74ad1d 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -240,15 +240,28 @@ calc as ( select case when sum(calc.tst) < 2 then 1/0 else 1 end pass from calc; -/* Simple test to make sure we have some MS-DRGs. -TODO: Rewrite this as the new-style test (insert into test_cases): ENC_L3_DRG -*/ -select case when unique_drgs < 10 then 1/0 else 1 end many_different_drgs from ( - select count(distinct drg) unique_drgs - from ( - select drg from encounter where drg is not null - ) - ); +insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) +select 'ENC_L3_DRG' query_name + , 'make sure we have some MS-DRGs' description + , case when drg = 'NULL or missing' and record_pct < 95 then 1 + when drg != 'NULL or missing' and record_n >= 1 then 1 + else 0 + end pass + , rownum obs + , q.* +from ( +with enc as ( +select coalesce(drg, 'NULL or missing') drg +from encounter +), +agg as ( + select count(*) tot from encounter) +select drg, count(*) record_n, round(count(*) / agg.tot * 100, 4) record_pct +from enc cross join agg +group by drg, agg.tot +order by 2 desc +) q +; insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) From 15107556413cc7467922e99b0230ffd253f20e61 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 16 Mar 2016 10:48:27 -0500 Subject: [PATCH 088/507] oops: VIT_L3_SMOKING only had the select part of the insert --- Oracle/cdm_transform_tests.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index df92d13..d72f5d8 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -198,7 +198,8 @@ from by_time join by_val on by_time.admit_ym = by_val.admit_ym ; - +insert into test_cases (query_name, description, pass + , obs, by_value1, by_value2, record_n, record_pct, distinct_patid_n) select 'VIT_L3_SMOKING' query_name , 'make sure we have something about patient smoking tobacco use' description , case From de33a62348e93288489cf7f8846c8ba77f8df9b8 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 16 Mar 2016 15:44:05 -0500 Subject: [PATCH 089/507] Post-process CDM (fill in provider id for diagnosis, procedures, convert units, etc.). --- Oracle/cdm_postproc.sql | 27 +++++++++++++++++++++++++++ Oracle/run-i2p-transform.sh | 3 +++ 2 files changed, 30 insertions(+) create mode 100644 Oracle/cdm_postproc.sql diff --git a/Oracle/cdm_postproc.sql b/Oracle/cdm_postproc.sql new file mode 100644 index 0000000..99ed4e3 --- /dev/null +++ b/Oracle/cdm_postproc.sql @@ -0,0 +1,27 @@ +/* Post-processing tasks against the CDM +*/ + +/* Copy providerid from encounter table to diagnosis, procedures tables. +CDM specification says: + "Please note: This is a field replicated from the ENCOUNTER table." +*/ +merge into diagnosis d +using encounter e + on (d.encounterid = e.encounterid) +when matched then update set d.providerid = e.providerid; + +merge into procedures p +using encounter e + on (p.encounterid = e.encounterid) +when matched then update set p.providerid = e.providerid; + +/* Currently in HERON, systolic and diastolic blood pressures from flowsheets +are reversed. This is fixed in the HERON ETL code (#4026) but we haven't run a +production ETL yet. +*/ +update vital v set v.systolic = v.diastolic, v.diastolic = v.systolic; + +/* Currently in HERON, we have hight in cm and weight in kg - the CDM wants +height in inches and weight in pounds. */ +update vital v set v.ht = v.ht * 0.393701; +update vital v set v.wt = v.wt * 2.20462; diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index b5b6657..93a2e90 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -108,6 +108,9 @@ start gather_table_stats.sql -- SCILHS transform start PCORNetLoader_ora.sql +-- Post-process steps +start cdm_postproc.sql + -- CDM transform tests start cdm_transform_tests.sql From b0008dfd5fc2ce4c92d6147c0416f1c2b3586075 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 16 Mar 2016 16:07:30 -0500 Subject: [PATCH 090/507] Dividing by 2.54 is likely more clear. Thanks to Dan for review. --- Oracle/cdm_postproc.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/cdm_postproc.sql b/Oracle/cdm_postproc.sql index 99ed4e3..854cfeb 100644 --- a/Oracle/cdm_postproc.sql +++ b/Oracle/cdm_postproc.sql @@ -23,5 +23,5 @@ update vital v set v.systolic = v.diastolic, v.diastolic = v.systolic; /* Currently in HERON, we have hight in cm and weight in kg - the CDM wants height in inches and weight in pounds. */ -update vital v set v.ht = v.ht * 0.393701; +update vital v set v.ht = v.ht / 2.54; update vital v set v.wt = v.wt * 2.20462; From f8161fe98938bfa251fe4b60b47fad701e926621 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 16 Mar 2016 18:01:16 -0500 Subject: [PATCH 091/507] Removed obsolete code that was replaced by merge statement for inout_cd. --- Oracle/heron_encounter_style.sql | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 06a6ce4..3b146a0 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -149,26 +149,6 @@ when matched then update set inout_cd = et.pcori_code; -- 11,085,911 rows merged. - -create index enc_type_codes_encnum_idx on enc_type(encounter_num); - -update "&&i2b2_data_schema".visit_dimension vd -set inout_cd = coalesce(( - select et.pcori_code from enc_type et - where vd.encounter_num = et.encounter_num - ), 'UN'); - --- Make sure the update went as expected -select case when count(*) > 0 then 1/0 else 1 end inout_cd_update_match from ( - select * from ( - select vd.inout_cd vd_code, et.pcori_code et_code - from "&&i2b2_data_schema".visit_dimension vd - left join enc_type et on et.encounter_num = vd.encounter_num - ) - where vd_code != vd_code and not (vd_code = 'UN' and vd_code is null) - ); - - /* Discharge disposition Valid values are: From 0e9f07e0b95fdbf5c9fd5fdd10c6c940943d6129 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 16 Mar 2016 18:30:39 -0500 Subject: [PATCH 092/507] Set timing on. --- Oracle/run-i2p-transform.sh | 1 + Oracle/update_to_cdm_dimensions.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 93a2e90..77d3c12 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -69,6 +69,7 @@ sqlplus /nolog < Date: Thu, 17 Mar 2016 11:47:24 -0500 Subject: [PATCH 093/507] Added *.pyc pattern to .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 872921f..0737f94 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.exe *.o *.so +*.pyc # Temporary files # ################### From 042e3fbd3dbaa65bd8666b1c43d52f127d6d2450 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 17 Mar 2016 14:09:16 -0500 Subject: [PATCH 094/507] Adjusted the smoking/tobacco use mapping. --- Oracle/pcornet_mapping.csv | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index d9de85e..2842c55 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -35,11 +35,10 @@ pcori_path,local_path \PCORI\VITAL\BP\SYSTOLIC\,\i2b2\Flowsheet\KU\GEN TMP CARD ECHO VITALS T:900426\KU GEN CARD GRP CV COMMON VITALS G:910000 I#2\BLOOD PRESSURE M:5 I#2\SYSTOLIC\ \PCORI\VITAL\ORIGINAL_BMI\,\i2b2\Flowsheet\MODEL\VITAL SIGNS COMPLEX T:31010\KU IP GRP ADULT HEIGHT/WEIGHT G:300220 I#4\KU IP ROW BMI M:301070 I#6\ \PCORI\VITAL\WT\,\i2b2\Flowsheet\KU\GEN CARD\IP/OP HEIGHT/WEIGHT T:900445\KU GEN CARD G IP/OP PROCEDURE HEIGHT/WEI G:900612 I#1\WEIGHT/SCALE M:14 I#2\ -\PCORI\VITAL\TOBACCO\SMOKING\YES\01\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Current Every Day Smoker\ -\PCORI\VITAL\TOBACCO\SMOKING\YES\02\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Current Some Day Smoker\ -\PCORI\VITAL\TOBACCO\SMOKING\03\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Former Smoker\ -\PCORI\VITAL\TOBACCO\SMOKING\04\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Never Smoker \ -\PCORI\VITAL\TOBACCO\SMOKING\YES\05\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Smoker, Current Status Unknown\ +\PCORI\VITAL\TOBACCO\02\01\,\i2b2\History\Social History\Tobacco Usage\Smokeless Tobacco Use\Current User\ +\PCORI\VITAL\TOBACCO\02\02\,\i2b2\History\Social History\Tobacco Usage\Smokeless Tobacco Use\Never Used\ +\PCORI\VITAL\TOBACCO\02\03\,\i2b2\History\Social History\Tobacco Usage\Smokeless Tobacco Use\Former User\ +\PCORI\VITAL\TOBACCO\SMOKING\YES\05\,"\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Smoker, Current Status Unknown\" \PCORI\VITAL\TOBACCO\SMOKING\06\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Unknown If Ever Smoked\ \PCORI\VITAL\TOBACCO\SMOKING\YES\07\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Heavy Tobacco Smoker\ \PCORI\VITAL\TOBACCO\SMOKING\YES\08\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Light Tobacco Smoker\ From 20809762fa712b3ac8a256e96ede7ad407a1bbe1 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Thu, 17 Mar 2016 16:06:42 -0500 Subject: [PATCH 095/507] Added test report for admitting source. --- Oracle/cdm_transform_tests.sql | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 08755d4..af13f8c 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -411,6 +411,35 @@ order by 1 ; +/* Should be populated for Inpatient Hospital Stay (IP) and Non-Acute +Institutional Stay (IS) encounter types. +*/ +insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) +select 'ENC_L3_ADMSRC' query_name + , 'too many NI' description + , case when admitting_source = 'NI' and record_pct < 40 then 1 + when admitting_source is null and record_pct < 5 then 1 + when admitting_source != 'NI' then 1 + else 0 + end pass + , rownum obs + , q.* +from ( + with ipis as ( + select * from encounter where enc_type in ('IP', 'IS') + ), + enc_agg as ( + select count(*) tot from ipis + ) + select admitting_source + , count(*) record_n + , round(count(*) / enc_agg.tot * 100, 4) record_pct + from ipis cross join enc_agg + group by admitting_source, enc_agg.tot + order by 1 + ) q +; + /* Due to using hostpital accounts as encounters, we have a long tail of very long encounters; hundreds of days. From cb5f77f35cdf06720d8d29371148b641694304af Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Thu, 17 Mar 2016 16:11:59 -0500 Subject: [PATCH 096/507] Populate admitting_source in encounter. - Added PCORI to local i2b2 path mappings for admitting source. - Populate visit dimension with admitting_source column based on observation facts. - Un-comment admitting source related code in the CDM transform that was temporarily commented out to avoid Oracle errors. - Insert local admitting source terms as leaves under PCORNet parents in the ontology. - Add variables expected by the dimension update script. --- Oracle/PCORNetLoader_ora.sql | 17 ++--------- Oracle/heron_encounter_style.sql | 48 ++++++++++++++++++++++++++++++ Oracle/pcornet_mapping.csv | 22 ++++++++++++++ Oracle/pcornet_mapping.sql | 34 +++++++++++++++++++++ Oracle/update_to_cdm_dimensions.sh | 3 ++ 5 files changed, 109 insertions(+), 15 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 5ca3e23..dcf97e6 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -947,20 +947,6 @@ end PCORNetDemographic; /* TODOs: - -1) -Figure out what "_source" is supposed to be. I'ts an invalid name in -oracle as-is. And, I don't see where it comes from - the tables were selecting -from don't have it. - -Error(16,266): PL/SQL: ORA-00911: invalid character -00911. 00000 - "invalid character" -*Cause: identifiers may not start with any ASCII character other than - letters and numbers. $#_ are also allowed after the first - character. - -Commented out that and what it's used for (ADMITTING_SOURCE) - 4) ORA-00904: "FACILITY_ID": invalid identifier @@ -987,7 +973,8 @@ select distinct v.patient_num, v.encounter_num, 'NI' facility_id, /* See TODO above */ CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END, CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END, - drg.drg, drg_type, 'NI' admitting_source /* see TODO above CASE WHEN _source IS NULL THEN 'NI' ELSE admitting_source END*/ + drg.drg, drg_type, + CASE WHEN admitting_source IS NULL THEN 'NI' ELSE admitting_source END admitting_source from i2b2visit v inner join demographic d on v.patient_num=d.patid left outer join (select * from diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 3b146a0..034f79e 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -270,6 +270,54 @@ set discharge_status = coalesce(( ), 'UN'); +/* Admitting source +There may be multiple per encounter - rank and pick one. +*/ +whenever sqlerror continue; +alter table "&&i2b2_data_schema".visit_dimension add ( + admitting_source VARCHAR2(4 BYTE) + ); +whenever sqlerror exit; + +merge into "&&i2b2_data_schema".visit_dimension vd using ( + with codes as ( + select + pm.pcori_path, cd.concept_cd, cd.name_char, + replace(pe.pcori_basecode, 'ADMITTING_SOURCE:', '') pcori_code + from + pcornet_mapping pm + join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' + join "&&i2b2_meta_schema".pcornet_enc pe on pe.c_fullname = pm.pcori_path + where pm.pcori_path like '\PCORI\ENCOUNTER\ADMITTING_SOURCE\%' + ), + enc_as as ( + select obs.encounter_num, codes.pcori_code + from "&&i2b2_data_schema".observation_fact obs + join codes on codes.concept_cd = obs.concept_cd + ), + ranks as ( + select + encounter_num, pcori_code, + -- Prioritize status: Last OT, UN, NI + decode(pcori_code, 'AF',0,'AL',1,'AV',2,'ED',3,'HH',4,'HO',5,'HS',6,'IP',7, + 'NH',8,'RH',9,'RS',10,'SN',11,'OT',12,'UN',13,'NI',14, 99) as_rank + from enc_as + ), + priority_rank as ( + select min(as_rank) as_rank, encounter_num + from ranks + group by encounter_num + ) + select distinct + ranks.encounter_num, coalesce(ranks.pcori_code, 'NI') pcori_code + from priority_rank pr + join ranks on pr.encounter_num = ranks.encounter_num and pr.as_rank = ranks.as_rank + ) admt_enc on (admt_enc.encounter_num = vd.encounter_num) +when matched then update +set vd.admitting_source = admt_enc.pcori_code +; + + /* -- Manually update encounter (i2p-transform should fill in the column) select count(*) from encounter; diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index d9de85e..145e191 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -194,3 +194,25 @@ pcori_path,local_path "\PCORI\ENCOUNTER\ENC_TYPE\IS\","\i2b2\Visit Details\Place of Service (IDX)\31\" "\PCORI\ENCOUNTER\ENC_TYPE\IS\","\i2b2\Visit Details\Place of Service (IDX)\54\" "\PCORI\ENCOUNTER\ENC_TYPE\AV\","\i2b2\Visit Details\Place of Service (IDX)\19\" +\PCORI\ENCOUNTER\ADMITTING_SOURCE\AV\,\i2b2\UHC\Visit Details\Admission Point Of Origin\21\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\AV\,\i2b2\UHC\Visit Details\Admission Point Of Origin\2\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\ED\,\i2b2\UHC\Visit Details\Admission Point Of Origin\7\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\HH\,\i2b2\UHC\Visit Details\Admission Point Of Origin\12\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\HH\,\i2b2\UHC\Visit Details\Admission Point Of Origin\11\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\HO\,\i2b2\UHC\Visit Details\Admission Point Of Origin\1\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\HS\,\i2b2\UHC\Visit Details\Admission Point Of Origin\22\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\IP\,\i2b2\UHC\Visit Details\Admission Point Of Origin\4\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\IP\,\i2b2\UHC\Visit Details\Admission Point Of Origin\10\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\IP\,\i2b2\UHC\Visit Details\Admission Point Of Origin\6\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\NI\,\i2b2\UHC\Visit Details\Admission Point Of Origin\9\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\NI\,\i2b2\UHC\Visit Details\Admission Point Of Origin\18\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\OT\,\i2b2\UHC\Visit Details\Admission Point Of Origin\13\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\OT\,\i2b2\UHC\Visit Details\Admission Point Of Origin\14\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\OT\,\i2b2\UHC\Visit Details\Admission Point Of Origin\15\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\OT\,\i2b2\UHC\Visit Details\Admission Point Of Origin\16\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\OT\,\i2b2\UHC\Visit Details\Admission Point Of Origin\19\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\OT\,\i2b2\UHC\Visit Details\Admission Point Of Origin\23\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\OT\,\i2b2\UHC\Visit Details\Admission Point Of Origin\24\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\OT\,\i2b2\UHC\Visit Details\Admission Point Of Origin\3\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\OT\,\i2b2\UHC\Visit Details\Admission Point Of Origin\8\ +\PCORI\ENCOUNTER\ADMITTING_SOURCE\SN\,\i2b2\UHC\Visit Details\Admission Point Of Origin\5\ diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 4c7df4b..145d3d8 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -268,3 +268,37 @@ local_drgs as ( select pd.drg_code, pd.c_name, ld.c_name from pcornet_drgs pd join local_drgs ld on ld.drg_code = pd.drg_code; */ + +/* Admitting source: For i2p-transform, these values go in the visit dimension. +To make them queryable from the web interface, insert the local terms under the +PCORI parent. +*/ + + +create or replace view admit_src_path_map as +select local_path, local_path || '%' like_local_path, pcori_path, pcori_path || '%' like_pcori_path +from pcornet_mapping where pcori_path like '\PCORI\ENCOUNTER\ADMITTING_SOURCE\%' +; + +update "&&i2b2_meta_schema".PCORNET_ENC +set c_visualattributes = 'FA' where c_fullname in ( + select pcori_path from admit_src_path_map + ); + +insert into "&&i2b2_meta_schema".PCORNET_ENC +select + pe.c_hlevel + 1 c_hlevel, + pm.pcori_path || ht.c_basecode || '\' c_fullname, + ht.c_name, ht.c_synonym_cd, ht.c_visualattributes, + ht.c_totalnum, ht.c_basecode, ht.c_metadataxml, ht.c_facttablecolumn, ht.c_tablename, + ht.c_columnname, ht.c_columndatatype, ht.c_operator, ht.c_dimcode, ht.c_comment, + ht.c_tooltip, ht.m_applied_path, ht.update_date, ht.download_date, ht.import_date, + 'MAPPING' sourcesystem_cd, ht.valuetype_cd, ht.m_exclusion_cd, ht.c_path, ht.c_symbol, + pe.pcori_basecode +from + "&&i2b2_meta_schema"."&&terms_table" ht +cross join admit_src_path_map pm +join "&&i2b2_meta_schema".pcornet_enc pe on pe.c_fullname = pm.pcori_path +where ht.c_fullname like pm.like_local_path +order by c_hlevel +; diff --git a/Oracle/update_to_cdm_dimensions.sh b/Oracle/update_to_cdm_dimensions.sh index 3ce94fb..2e8ca84 100644 --- a/Oracle/update_to_cdm_dimensions.sh +++ b/Oracle/update_to_cdm_dimensions.sh @@ -4,6 +4,8 @@ set -e # Expected environment variables (put there by Jenkins, etc.) # export pcornet_cdm_user= # export pcornet_cdm= +# export i2b2_data_schema= +# export i2b2_meta_schema= sqlplus /nolog < Date: Thu, 17 Mar 2016 16:23:09 -0500 Subject: [PATCH 097/507] Fixed mapping mistake. Replaced the wrong lines. --- Oracle/pcornet_mapping.csv | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 2842c55..c41a3f2 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -35,17 +35,17 @@ pcori_path,local_path \PCORI\VITAL\BP\SYSTOLIC\,\i2b2\Flowsheet\KU\GEN TMP CARD ECHO VITALS T:900426\KU GEN CARD GRP CV COMMON VITALS G:910000 I#2\BLOOD PRESSURE M:5 I#2\SYSTOLIC\ \PCORI\VITAL\ORIGINAL_BMI\,\i2b2\Flowsheet\MODEL\VITAL SIGNS COMPLEX T:31010\KU IP GRP ADULT HEIGHT/WEIGHT G:300220 I#4\KU IP ROW BMI M:301070 I#6\ \PCORI\VITAL\WT\,\i2b2\Flowsheet\KU\GEN CARD\IP/OP HEIGHT/WEIGHT T:900445\KU GEN CARD G IP/OP PROCEDURE HEIGHT/WEI G:900612 I#1\WEIGHT/SCALE M:14 I#2\ -\PCORI\VITAL\TOBACCO\02\01\,\i2b2\History\Social History\Tobacco Usage\Smokeless Tobacco Use\Current User\ -\PCORI\VITAL\TOBACCO\02\02\,\i2b2\History\Social History\Tobacco Usage\Smokeless Tobacco Use\Never Used\ -\PCORI\VITAL\TOBACCO\02\03\,\i2b2\History\Social History\Tobacco Usage\Smokeless Tobacco Use\Former User\ -\PCORI\VITAL\TOBACCO\SMOKING\YES\05\,"\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Smoker, Current Status Unknown\" +\PCORI\VITAL\TOBACCO\SMOKING\YES\01\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Current Every Day Smoker\ +\PCORI\VITAL\TOBACCO\SMOKING\YES\02\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Current Some Day Smoker\ +\PCORI\VITAL\TOBACCO\SMOKING\03\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Former Smoker\ +\PCORI\VITAL\TOBACCO\SMOKING\04\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Never Smoker \ +\PCORI\VITAL\TOBACCO\SMOKING\YES\05\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Smoker, Current Status Unknown\ \PCORI\VITAL\TOBACCO\SMOKING\06\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Unknown If Ever Smoked\ \PCORI\VITAL\TOBACCO\SMOKING\YES\07\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Heavy Tobacco Smoker\ \PCORI\VITAL\TOBACCO\SMOKING\YES\08\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Light Tobacco Smoker\ -\PCORI\VITAL\TOBACCO\02\01\,\i2b2\History\Social History\Tobacco Usage\Yes\ -\PCORI\VITAL\TOBACCO\02\02\,\i2b2\History\Social History\Tobacco Usage\Never\ -\PCORI\VITAL\TOBACCO\02\03\,\i2b2\History\Social History\Tobacco Usage\Quit\ -\PCORI\VITAL\TOBACCO\02\04\,\i2b2\History\Social History\Tobacco Usage\Passive\ +\PCORI\VITAL\TOBACCO\02\01\,\i2b2\History\Social History\Tobacco Usage\Smokeless Tobacco Use\Current User\ +\PCORI\VITAL\TOBACCO\02\02\,\i2b2\History\Social History\Tobacco Usage\Smokeless Tobacco Use\Never Used\ +\PCORI\VITAL\TOBACCO\02\03\,\i2b2\History\Social History\Tobacco Usage\Smokeless Tobacco Use\Former User\ \PCORI\VITAL\TOBACCO\UN\06\,\i2b2\History\Social History\Tobacco Usage\Not Asked\ \PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\253\ \PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\A\,\i2b2\Visit Details\Discharge Disposition Codes\09\ From 1876f4fdcd2585a2808ae910073960b6db9cf620 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Thu, 17 Mar 2016 17:07:54 -0500 Subject: [PATCH 098/507] Delete custom mappings from pcornet encounter ontology before inserting. Thanks to Mike for the review. --- Oracle/pcornet_mapping.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 145d3d8..e17d591 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -10,6 +10,9 @@ set echo on; delete from "&&i2b2_meta_schema".PCORNET_VITAL where sourcesystem_cd='MAPPING'; +delete +from "&&i2b2_meta_schema".PCORNET_ENC where sourcesystem_cd='MAPPING'; + insert into "&&i2b2_meta_schema".PCORNET_VITAL SELECT PCORNET_VITAL.C_HLEVEL+1, PCORNET_VITAL.C_FULLNAME || i2b2.c_basecode || '\' as C_FULLNAME, From 18362c1b90b16ea45495ae7fa82e4e91021b4c99 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Thu, 17 Mar 2016 18:13:45 -0500 Subject: [PATCH 099/507] Factor out loading the pcornet mapping since it's needed by both the transform and the dimension update. --- Oracle/load_pcornet_mapping.sh | 29 +++++++++++++++++++++++++++++ Oracle/run-i2p-transform.sh | 19 +------------------ Oracle/update_to_cdm_dimensions.sh | 2 ++ 3 files changed, 32 insertions(+), 18 deletions(-) create mode 100644 Oracle/load_pcornet_mapping.sh diff --git a/Oracle/load_pcornet_mapping.sh b/Oracle/load_pcornet_mapping.sh new file mode 100644 index 0000000..651d6cc --- /dev/null +++ b/Oracle/load_pcornet_mapping.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e + +# Expected environment variables +# Database SID +# export sid= + +# User and password for CDM user +# export pcornet_cdm_user= +# export pcornet_cdm= + +# Create/Load the local path mapping and ontology update tables +sqlplus /nolog < Date: Wed, 23 Mar 2016 09:35:01 -0500 Subject: [PATCH 100/507] Use .csv file for site-specific harvest columns. - Load harvest_local.csv as part of the transform shell script. --- Oracle/PCORNetLoader_ora.sql | 3 ++- Oracle/harvest_local.csv | 2 ++ Oracle/harvest_local.ctl | 28 ++++++++++++++++++++++++++++ Oracle/harvest_local_ddl.sql | 22 ++++++++++++++++++++++ Oracle/load_harvest_local.sh | 26 ++++++++++++++++++++++++++ Oracle/run-i2p-transform.sh | 1 + 6 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 Oracle/harvest_local.csv create mode 100644 Oracle/harvest_local.ctl create mode 100644 Oracle/harvest_local_ddl.sql create mode 100644 Oracle/load_harvest_local.sh diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index dcf97e6..822197f 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1397,7 +1397,8 @@ create or replace procedure PCORNetHarvest as begin INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) - VALUES('&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, 01, 02, 1,1,2,1,2,1,2,1,2,1,1,2,2,1,1,1,2,1,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,null,current_date,null,null,null); + select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.PX_DATE_MGMT, hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,null,current_date,null,null,null + from harvest_local hl; end PCORNetHarvest; / diff --git a/Oracle/harvest_local.csv b/Oracle/harvest_local.csv new file mode 100644 index 0000000..e20719b --- /dev/null +++ b/Oracle/harvest_local.csv @@ -0,0 +1,2 @@ +"DATAMART_CLAIMS","DATAMART_EHR","BIRTH_DATE_MGMT","ENR_START_DATE_MGMT","ENR_END_DATE_MGMT","ADMIT_DATE_MGMT","DISCHARGE_DATE_MGMT","PX_DATE_MGMT","RX_ORDER_DATE_MGMT","RX_START_DATE_MGMT","RX_END_DATE_MGMT","DISPENSE_DATE_MGMT","LAB_ORDER_DATE_MGMT","SPECIMEN_DATE_MGMT","RESULT_DATE_MGMT","MEASURE_DATE_MGMT","ONSET_DATE_MGMT","REPORT_DATE_MGMT","RESOLVE_DATE_MGMT","PRO_DATE_MGMT" +"01","02","03","03","03","03","03","03","03","03","04","03","03","03","03","03","03","03","03","03" diff --git a/Oracle/harvest_local.ctl b/Oracle/harvest_local.ctl new file mode 100644 index 0000000..91726a7 --- /dev/null +++ b/Oracle/harvest_local.ctl @@ -0,0 +1,28 @@ +options (errors=0, skip=1) +load data +truncate into table harvest_local +fields terminated by ',' optionally enclosed by '"' +trailing nullcols( + DATAMART_CLAIMS, + DATAMART_EHR, + BIRTH_DATE_MGMT, + ENR_START_DATE_MGMT, + ENR_END_DATE_MGMT, + ADMIT_DATE_MGMT, + DISCHARGE_DATE_MGMT, + PX_DATE_MGMT, + RX_ORDER_DATE_MGMT, + RX_START_DATE_MGMT, + RX_END_DATE_MGMT, + DISPENSE_DATE_MGMT, + LAB_ORDER_DATE_MGMT, + SPECIMEN_DATE_MGMT, + RESULT_DATE_MGMT, + MEASURE_DATE_MGMT, + ONSET_DATE_MGMT, + REPORT_DATE_MGMT, + RESOLVE_DATE_MGMT, + PRO_DATE_MGMT + ) + + diff --git a/Oracle/harvest_local_ddl.sql b/Oracle/harvest_local_ddl.sql new file mode 100644 index 0000000..ab376ca --- /dev/null +++ b/Oracle/harvest_local_ddl.sql @@ -0,0 +1,22 @@ +CREATE TABLE harvest_local ( + "DATAMART_CLAIMS" VARCHAR2(4), + "DATAMART_EHR" VARCHAR2(4), + "BIRTH_DATE_MGMT" VARCHAR2(4), + "ENR_START_DATE_MGMT" VARCHAR2(4), + "ENR_END_DATE_MGMT" VARCHAR2(4), + "ADMIT_DATE_MGMT" VARCHAR2(4), + "DISCHARGE_DATE_MGMT" VARCHAR2(4), + "PX_DATE_MGMT" VARCHAR2(4), + "RX_ORDER_DATE_MGMT" VARCHAR2(4), + "RX_START_DATE_MGMT" VARCHAR2(4), + "RX_END_DATE_MGMT" VARCHAR2(4), + "DISPENSE_DATE_MGMT" VARCHAR2(4), + "LAB_ORDER_DATE_MGMT" VARCHAR2(4), + "SPECIMEN_DATE_MGMT" VARCHAR2(4), + "RESULT_DATE_MGMT" VARCHAR2(4), + "MEASURE_DATE_MGMT" VARCHAR2(4), + "ONSET_DATE_MGMT" VARCHAR2(4), + "REPORT_DATE_MGMT" VARCHAR2(4), + "RESOLVE_DATE_MGMT" VARCHAR2(4), + "PRO_DATE_MGMT" VARCHAR2(4) + ); diff --git a/Oracle/load_harvest_local.sh b/Oracle/load_harvest_local.sh new file mode 100644 index 0000000..9ba8721 --- /dev/null +++ b/Oracle/load_harvest_local.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e + +# Expected environment variables (put there by Jenkins, etc.) + +# Database SID +# export sid= + +# User and password for CDM user +# export pcornet_cdm_user= +# export pcornet_cdm= + +# Create/Load the local harvest data +sqlplus /nolog < Date: Wed, 23 Mar 2016 10:12:27 -0500 Subject: [PATCH 101/507] Only fill in refresh dates in the harvest table if there are rows in the corresponding CDM tables. - As per the PCORNet CDM v3 specification. --- Oracle/PCORNetLoader_ora.sql | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 822197f..5d4ddcf 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1397,7 +1397,21 @@ create or replace procedure PCORNetHarvest as begin INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) - select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.PX_DATE_MGMT, hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,null,current_date,null,null,null + select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.PX_DATE_MGMT, hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, + case when (select count(*) from demographic) > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, + case when (select count(*) from enrollment) > 0 then current_date else null end REFRESH_ENROLLMENT_DATE, + case when (select count(*) from encounter) > 0 then current_date else null end REFRESH_ENCOUNTER_DATE, + case when (select count(*) from diagnosis) > 0 then current_date else null end REFRESH_DIAGNOSIS_DATE, + case when (select count(*) from procedures) > 0 then current_date else null end REFRESH_PROCEDURES_DATE, + case when (select count(*) from vital) > 0 then current_date else null end REFRESH_VITAL_DATE, + case when (select count(*) from dispensing) > 0 then current_date else null end REFRESH_DISPENSING_DATE, + case when (select count(*) from lab_result_cm) > 0 then current_date else null end REFRESH_LAB_RESULT_CM_DATE, + case when (select count(*) from condition) > 0 then current_date else null end REFRESH_CONDITION_DATE, + case when (select count(*) from pro_cm) > 0 then current_date else null end REFRESH_PRO_CM_DATE, + case when (select count(*) from prescribing) > 0 then current_date else null end REFRESH_PRESCRIBING_DATE, + case when (select count(*) from pcornet_trial) > 0 then current_date else null end REFRESH_PCORNET_TRIAL_DATE, + case when (select count(*) from death) > 0 then current_date else null end REFRESH_DEATH_DATE, + case when (select count(*) from death_cause) > 0 then current_date else null end REFRESH_DEATH_CAUSE_DATE from harvest_local hl; end PCORNetHarvest; From 8b0a3ef4458e53ba4887d8b6de91ac08adc6a700 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 23 Mar 2016 17:46:33 -0500 Subject: [PATCH 102/507] Python script to generate .ctl, .sql from .csv and load .csv - Load local harvest data using new python script. --- Oracle/harvest_local.ctl | 28 ------ Oracle/harvest_local_ddl.sql | 22 ----- Oracle/load_csv.py | 179 +++++++++++++++++++++++++++++++++++ Oracle/load_harvest_local.sh | 26 ----- 4 files changed, 179 insertions(+), 76 deletions(-) delete mode 100644 Oracle/harvest_local.ctl delete mode 100644 Oracle/harvest_local_ddl.sql create mode 100644 Oracle/load_csv.py delete mode 100644 Oracle/load_harvest_local.sh diff --git a/Oracle/harvest_local.ctl b/Oracle/harvest_local.ctl deleted file mode 100644 index 91726a7..0000000 --- a/Oracle/harvest_local.ctl +++ /dev/null @@ -1,28 +0,0 @@ -options (errors=0, skip=1) -load data -truncate into table harvest_local -fields terminated by ',' optionally enclosed by '"' -trailing nullcols( - DATAMART_CLAIMS, - DATAMART_EHR, - BIRTH_DATE_MGMT, - ENR_START_DATE_MGMT, - ENR_END_DATE_MGMT, - ADMIT_DATE_MGMT, - DISCHARGE_DATE_MGMT, - PX_DATE_MGMT, - RX_ORDER_DATE_MGMT, - RX_START_DATE_MGMT, - RX_END_DATE_MGMT, - DISPENSE_DATE_MGMT, - LAB_ORDER_DATE_MGMT, - SPECIMEN_DATE_MGMT, - RESULT_DATE_MGMT, - MEASURE_DATE_MGMT, - ONSET_DATE_MGMT, - REPORT_DATE_MGMT, - RESOLVE_DATE_MGMT, - PRO_DATE_MGMT - ) - - diff --git a/Oracle/harvest_local_ddl.sql b/Oracle/harvest_local_ddl.sql deleted file mode 100644 index ab376ca..0000000 --- a/Oracle/harvest_local_ddl.sql +++ /dev/null @@ -1,22 +0,0 @@ -CREATE TABLE harvest_local ( - "DATAMART_CLAIMS" VARCHAR2(4), - "DATAMART_EHR" VARCHAR2(4), - "BIRTH_DATE_MGMT" VARCHAR2(4), - "ENR_START_DATE_MGMT" VARCHAR2(4), - "ENR_END_DATE_MGMT" VARCHAR2(4), - "ADMIT_DATE_MGMT" VARCHAR2(4), - "DISCHARGE_DATE_MGMT" VARCHAR2(4), - "PX_DATE_MGMT" VARCHAR2(4), - "RX_ORDER_DATE_MGMT" VARCHAR2(4), - "RX_START_DATE_MGMT" VARCHAR2(4), - "RX_END_DATE_MGMT" VARCHAR2(4), - "DISPENSE_DATE_MGMT" VARCHAR2(4), - "LAB_ORDER_DATE_MGMT" VARCHAR2(4), - "SPECIMEN_DATE_MGMT" VARCHAR2(4), - "RESULT_DATE_MGMT" VARCHAR2(4), - "MEASURE_DATE_MGMT" VARCHAR2(4), - "ONSET_DATE_MGMT" VARCHAR2(4), - "REPORT_DATE_MGMT" VARCHAR2(4), - "RESOLVE_DATE_MGMT" VARCHAR2(4), - "PRO_DATE_MGMT" VARCHAR2(4) - ); diff --git a/Oracle/load_csv.py b/Oracle/load_csv.py new file mode 100644 index 0000000..1e18b58 --- /dev/null +++ b/Oracle/load_csv.py @@ -0,0 +1,179 @@ +''' load_csv.py - Load a specified data file with sqlplus/sqlldr +''' +from contextlib import contextmanager +from collections import defaultdict +from csv import DictReader +from functools import partial +import logging + +# OCAP exception for logging +logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', + datefmt='%Y.%m.%d %H:%M:%S', level=logging.INFO) +log = logging.getLogger(__name__) + + +def main(argv, environ, open_argv, mk_sqlplus): + def logif(logf, s): + if s: + logf(s) + + sp = mk_sqlplus() + + # TODO: Consider docopt + table_name, csv, ctl_out_fn, user_env, password_env = argv[1:6] + + # Use Oracle utilities to drop/create table based on csv input + with open_argv(csv, 'rb') as fin: + dr = DictReader(fin) + ret, so, se = sp.create_table(user=environ[user_env], + password=environ[password_env], + table_name=table_name, + dr=dr) + logif(log.info, so) + logif(log.error, se) + if ret: + raise RuntimeError() + + # Write out a control file based on csv input + with open_argv(ctl_out_fn, 'wb') as ctl_fout: + ctl_fout.write(sp.ctl_from_csv(schema_table=table_name, + fields=dr.fieldnames)) + + # Load .csv using Oracle utilities + ret, so, se = sp.load_table(user=environ[user_env], + password=environ[password_env], + table_name=table_name, + ctl_fn=ctl_out_fn, + csv=csv) + logif(log.info, so) + logif(log.error, se) + if ret: + raise RuntimeError() + + +class MockPopen(object): + def __init__(self, cmd_args, stdout=None, stderr=None, shell=False): + self ._cmd_args = cmd_args + + def communicate(self): + log.info('MockPopen::communicate: %s' % (self._cmd_args)) + return ('', '') + + +class OracleUtils(object): + ''' + Test DDL generation - note test for 0-length column + >>> from StringIO import StringIO + >>> sio = StringIO('\\n'.join([l.strip() for l in """col1,col2,col3 + ... some data,Some other data and stuff,""".split('\\n')])) + >>> s = OracleUtils(MockPopen, None) + >>> print s.ddl_from_csv('some_table', DictReader(sio)) + ... # doctest: +NORMALIZE_WHITESPACE + create table some_table ( + col1 varchar2(16), + col2 varchar2(32), + col3 varchar2(16) + ); + + Test control file generation + >>> print s.ctl_from_csv('some_table', ['col1', 'col2']) + ... # doctest: +NORMALIZE_WHITESPACE + options (errors=0, skip=1) + load data + truncate into table some_table + fields terminated by ',' optionally enclosed by '"' + trailing nullcols( + col1, + col2 + ) + ''' + def __init__(self, Popen, PIPE, sqlplus_cmd='sqlplus /nolog', + sqlldr_cmd='sqlldr'): + self._sqlplus_cmd = sqlplus_cmd + self._sqlldr_cmd = sqlldr_cmd + self._Popen = Popen + self.PIPE = PIPE + + @classmethod + def make(cls, Popen, PIPE): + return OracleUtils(Popen, PIPE) + + def create_table(self, user, password, table_name, dr): + # Note: shell = true might not be the most secure, but it makes + # piplining easier. + # See also https://docs.python.org/2.7/library/subprocess.html#replacing-shell-pipeline # noqa + # Also, be warned taht SQL injection is possible with arguments! + cmd = self._sqlplus_cmd + ' ' + ( + '\n'.join([c.strip() for c in """< Date: Wed, 23 Mar 2016 18:59:05 -0500 Subject: [PATCH 103/507] use single-table TableTool - reduce authority from OracleUtils to TableTool - rename mk_sqplus - factor out run() - move logif idiom to run() - factor out dedent() - let main() code speak for itself - sort imports - PIPE is powerless; ok to import normally PEP8 whitespace tweak (excuse the non-atomic commit) avoid overly dynamic **_tcb() idiom --- Oracle/load_csv.py | 167 ++++++++++++++++++++++----------------------- 1 file changed, 80 insertions(+), 87 deletions(-) diff --git a/Oracle/load_csv.py b/Oracle/load_csv.py index 1e18b58..c2fef6b 100644 --- a/Oracle/load_csv.py +++ b/Oracle/load_csv.py @@ -1,9 +1,9 @@ ''' load_csv.py - Load a specified data file with sqlplus/sqlldr ''' -from contextlib import contextmanager from collections import defaultdict +from contextlib import contextmanager from csv import DictReader -from functools import partial +from subprocess import PIPE import logging # OCAP exception for logging @@ -12,62 +12,43 @@ log = logging.getLogger(__name__) -def main(argv, environ, open_argv, mk_sqlplus): - def logif(logf, s): - if s: - logf(s) - - sp = mk_sqlplus() - +def main(argv, environ, open_argv, mk_tool): # TODO: Consider docopt table_name, csv, ctl_out_fn, user_env, password_env = argv[1:6] - # Use Oracle utilities to drop/create table based on csv input + table_tool = mk_tool(table_name=table_name, + user=environ[user_env], + password=environ[password_env]) + with open_argv(csv, 'rb') as fin: dr = DictReader(fin) - ret, so, se = sp.create_table(user=environ[user_env], - password=environ[password_env], - table_name=table_name, - dr=dr) - logif(log.info, so) - logif(log.error, se) - if ret: - raise RuntimeError() - - # Write out a control file based on csv input + table_tool.create(dr=dr) + with open_argv(ctl_out_fn, 'wb') as ctl_fout: - ctl_fout.write(sp.ctl_from_csv(schema_table=table_name, - fields=dr.fieldnames)) + ctl_code = table_tool.ctl_from_csv(fields=dr.fieldnames) + ctl_fout.write(ctl_code) - # Load .csv using Oracle utilities - ret, so, se = sp.load_table(user=environ[user_env], - password=environ[password_env], - table_name=table_name, - ctl_fn=ctl_out_fn, - csv=csv) - logif(log.info, so) - logif(log.error, se) - if ret: - raise RuntimeError() + table_tool.load(ctl_fn=ctl_out_fn, csv=csv) class MockPopen(object): def __init__(self, cmd_args, stdout=None, stderr=None, shell=False): - self ._cmd_args = cmd_args + self._cmd_args = cmd_args + self.returncode = 0 def communicate(self): log.info('MockPopen::communicate: %s' % (self._cmd_args)) return ('', '') -class OracleUtils(object): +class TableTool(object): ''' Test DDL generation - note test for 0-length column >>> from StringIO import StringIO >>> sio = StringIO('\\n'.join([l.strip() for l in """col1,col2,col3 ... some data,Some other data and stuff,""".split('\\n')])) - >>> s = OracleUtils(MockPopen, None) - >>> print s.ddl_from_csv('some_table', DictReader(sio)) + >>> s = TableTool('me', 'sekret', 'some_table', MockPopen) + >>> print s.ddl_from_csv(DictReader(sio)) ... # doctest: +NORMALIZE_WHITESPACE create table some_table ( col1 varchar2(16), @@ -76,7 +57,7 @@ class OracleUtils(object): ); Test control file generation - >>> print s.ctl_from_csv('some_table', ['col1', 'col2']) + >>> print s.ctl_from_csv(['col1', 'col2']) ... # doctest: +NORMALIZE_WHITESPACE options (errors=0, skip=1) load data @@ -87,80 +68,90 @@ class OracleUtils(object): col2 ) ''' - def __init__(self, Popen, PIPE, sqlplus_cmd='sqlplus /nolog', + def __init__(self, Popen, user, password, table_name, + sqlplus_cmd='sqlplus /nolog', sqlldr_cmd='sqlldr'): - self._sqlplus_cmd = sqlplus_cmd - self._sqlldr_cmd = sqlldr_cmd - self._Popen = Popen - self.PIPE = PIPE + def run(template): + cmd = dedent(template.format( + sqlplus=sqlplus_cmd, sqlldr=sqlldr_cmd, + user=user, password=password, + table_name=table_name)) + p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) + stdout, stderr = p.communicate() + if stdout: + log.info('stdout: %s', stdout) + if stderr: + log.warn('stderr: %s', stderr) + if p.returncode: + raise IOError(p.returncode) + + self.run = run + self.table_name = table_name @classmethod - def make(cls, Popen, PIPE): - return OracleUtils(Popen, PIPE) + def make(cls, Popen, user, password, name): + return TableTool(Popen, user, password, name) - def create_table(self, user, password, table_name, dr): + def create(self, dr): # Note: shell = true might not be the most secure, but it makes # piplining easier. # See also https://docs.python.org/2.7/library/subprocess.html#replacing-shell-pipeline # noqa - # Also, be warned taht SQL injection is possible with arguments! - cmd = self._sqlplus_cmd + ' ' + ( - '\n'.join([c.strip() for c in """< Date: Thu, 24 Mar 2016 08:49:11 -0500 Subject: [PATCH 104/507] Properly quoted CSV element containing a comma. --- Oracle/pcornet_mapping.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index c41a3f2..2f16662 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -39,7 +39,7 @@ pcori_path,local_path \PCORI\VITAL\TOBACCO\SMOKING\YES\02\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Current Some Day Smoker\ \PCORI\VITAL\TOBACCO\SMOKING\03\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Former Smoker\ \PCORI\VITAL\TOBACCO\SMOKING\04\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Never Smoker \ -\PCORI\VITAL\TOBACCO\SMOKING\YES\05\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Smoker, Current Status Unknown\ +\PCORI\VITAL\TOBACCO\SMOKING\YES\05\,"\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Smoker, Current Status Unknown\" \PCORI\VITAL\TOBACCO\SMOKING\06\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Unknown If Ever Smoked\ \PCORI\VITAL\TOBACCO\SMOKING\YES\07\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Heavy Tobacco Smoker\ \PCORI\VITAL\TOBACCO\SMOKING\YES\08\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Light Tobacco Smoker\ From 4b7a3305c87217ce67675d113eb2e07165fa8ee7 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 24 Mar 2016 09:04:55 -0500 Subject: [PATCH 105/507] Whoops! Somewhere along the line I accidentally committed these merge files. --- Oracle/cdm_transform_tests_BACKUP_49608.sql | 180 -------------------- Oracle/cdm_transform_tests_BASE_49608.sql | 46 ----- Oracle/cdm_transform_tests_LOCAL_49608.sql | 83 --------- Oracle/cdm_transform_tests_REMOTE_49608.sql | 139 --------------- 4 files changed, 448 deletions(-) delete mode 100644 Oracle/cdm_transform_tests_BACKUP_49608.sql delete mode 100644 Oracle/cdm_transform_tests_BASE_49608.sql delete mode 100644 Oracle/cdm_transform_tests_LOCAL_49608.sql delete mode 100644 Oracle/cdm_transform_tests_REMOTE_49608.sql diff --git a/Oracle/cdm_transform_tests_BACKUP_49608.sql b/Oracle/cdm_transform_tests_BACKUP_49608.sql deleted file mode 100644 index 81fece0..0000000 --- a/Oracle/cdm_transform_tests_BACKUP_49608.sql +++ /dev/null @@ -1,180 +0,0 @@ -/* Test to make sure we got about the same number of patients in the CDM -diagnoses that we do in i2b2. -*/ -with cdm as ( - select count(distinct patid) qty from diagnosis - ), -i2b2 as ( - select count(distinct patient_num) qty from i2b2fact - where concept_cd in ( - select concept_cd from "&&i2b2_data_schema".concept_dimension - where concept_path like '\i2b2\Diagnoses\ICD9\%' - ) - ), -diff as ( - select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct from cdm cross join i2b2 - ) -select case when diff.pct > 10 then 1/0 else 1 end diag_pat_count_ok from diff; - - -/* Test to make sure we got about the same number of patients in the CDM -procedure that we do in i2b2. -*/ -with cdm as ( - select count(distinct patid) qty from procedures - ), -i2b2 as ( - select count(distinct patient_num) qty from ( - select * from i2b2fact - where concept_cd in ( - select concept_cd from "&&i2b2_data_schema".concept_dimension - where concept_path like '\i2b2\Procedures\%' - ) - ) - ), -diff as ( - select - ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct - from cdm cross join i2b2 - ) -select case when diff.pct > 10 then 1/0 else 1 end proc_pat_count_ok from diff; - - -/* Make sure we have roughly the same number of Hispanic patients in the CDM and -i2b2. -*/ -with -num_hispanic_cdm as ( - select count(*) qty from demographic where hispanic = 'Y' - ), -num_hispanic_i2b2 as ( - select count(*) qty from "&&i2b2_data_schema".patient_dimension where ethnicity_cd = 'Y' - ), -diff as ( - select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct - from num_hispanic_cdm cdm cross join num_hispanic_i2b2 i2b2 - ) -select case when diff.pct > 10 then 1/0 else 1 end hisp_y_pat_count_ok from diff; - --- TODO: Consider trying to combine the Y and N tests as they are copy/paste -with -num_hispanic_cdm as ( - select count(*) qty from demographic where hispanic = 'N' - ), -num_hispanic_i2b2 as ( - select count(*) qty from "&&i2b2_data_schema".patient_dimension where ethnicity_cd = 'N' - ), -diff as ( - select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct - from num_hispanic_cdm cdm cross join num_hispanic_i2b2 i2b2 - ) -select case when diff.pct > 10 then 1/0 else 1 end hisp_n_pat_count_ok from diff; - --- Make sure we have some diagnosis source information -select case when count(*) < 3 then 1/0 else 1 end a_few_dx_sources from ( - select distinct dx_source from diagnosis - ); - --- Make sure we have valid DX_SOURCE values -select case when count(*) > 0 then 1/0 else 1 end valid_dx_sources from ( - select distinct dx_source from diagnosis where dx_source not in ('AD', 'DI', 'FI', 'IN', 'NI', 'UN', 'OT') - ); - --- Make sure we have a couple principal diagnoses (PDX) -select case when count(*) < 2 then 1/0 else 1 end a_few_pdx_flags from ( - select distinct pdx from diagnosis - ); - --- Make sure we have valid PDX values -select case when count(*) > 0 then 1/0 else 1 end valid_pdx_flags from ( - select distinct pdx from diagnosis where pdx not in ('P', 'S', 'X', 'NI', 'UN', 'OT') - ); - --- Make sure that there are several different enrollment dates -select case when pct_distinct < 5 then 1/0 else 1 end many_enr_dates from ( - with all_enrs as ( - select count(*) qty from enrollment - ), - distinct_date as ( - select count(qty) qty from ( - select distinct enr_start_date qty from enrollment - ) - ) - select round((distinct_date.qty/all_enrs.qty) * 100, 4) pct_distinct - from distinct_date cross join all_enrs - ); - --- Make sure most procedure dates are not null -select case when pct_not_null < 99 then 1/0 else 1 end some_px_dates_not_null from ( - with all_px as ( - select count(*) qty from procedures - ), - not_null as ( - select count(*) qty from procedures where px_date is not null - ) - select round((not_null.qty / all_px.qty) * 100, 4) pct_not_null - from not_null cross join all_px -); - --- Make sure we have some procedure sources (px_source) -select case when count(*) = 0 then 1/0 else 1 end have_px_sources from ( - select distinct px_source from procedures where px_source is not null - ); -/* Test to make sure we have something about patient smoking tobacco use */ -with snums as ( - select smoking cat, count(smoking) qty from vital group by smoking -), -tot as ( - select sum(qty) as cnt from snums -), -calc as ( - select snums.cat, (snums.qty/tot.cnt*100) pct, - case when (snums.qty/tot.cnt*100) > 1 then 1 else 0 end tst - from snums, tot - where snums.cat!='NI' -) -<<<<<<< HEAD -select case when sum(calc.tst) < 3 then 1/0 else 1 end smoking_count_ok from calc; - - -/* Test to make sure we have something about patient general tobacco use */ -with tnums as ( - select tobacco cat, count(tobacco) qty from vital group by tobacco -), -tot as ( - select sum(qty) as cnt from tnums -), -calc as ( - select tnums.cat, (tnums.qty/tot.cnt*100) pct, case when (tnums.qty/tot.cnt*100) > 1 then 1 else 0 end tst from tnums, tot where tnums.cat!='NI' -) -select case when sum(calc.tst) < 3 then 1/0 else 1 end pass from calc; - - -/* Test to make sure we have something about tobacco use types */ -with ttnums as ( - select tobacco_type cat, count(tobacco) qty from vital group by tobacco_type order by cat -), -tot as ( - select sum(qty) as cnt from ttnums -), -calc as ( - select ttnums.cat, (ttnums.qty/tot.cnt*100) pct, case when (ttnums.qty/tot.cnt*100) > 1 then 1 else 0 end tst from ttnums, tot where ttnums.cat!='NI' -) -select case when sum(calc.tst) < 2 then 1/0 else 1 end pass from calc; - - -======= -select case when smokers.qty > 0 then 1 else 1/0 end smoker_count_ok from smokers; - --- Make sure we have some encounter types -select case when pct_known < 20 then 1/0 else 1 end some_known_enc_types from ( - with all_enc as ( - select count(*) qty from encounter - ), - known_enc as ( - select count(*) qty from encounter where enc_type is not null and enc_type != 'UN' - ) - select round((known_enc.qty / all_enc.qty) * 100, 4) pct_known - from known_enc cross join all_enc - ); ->>>>>>> upstream/master diff --git a/Oracle/cdm_transform_tests_BASE_49608.sql b/Oracle/cdm_transform_tests_BASE_49608.sql deleted file mode 100644 index 5d69cc3..0000000 --- a/Oracle/cdm_transform_tests_BASE_49608.sql +++ /dev/null @@ -1,46 +0,0 @@ -/* Test to make sure we got about the same number of patients in the CDM -diagnoses that we do in i2b2. -*/ -with cdm as ( - select count(distinct patid) qty from diagnosis - ), -i2b2 as ( - select count(distinct patient_num) qty from i2b2fact - where concept_cd in ( - select concept_cd from "&&i2b2_data_schema".concept_dimension - where concept_path like '\i2b2\Diagnoses\ICD9\%' - ) - ), -diff as ( - select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct from cdm cross join i2b2 - ) -select case when diff.pct > 10 then 1/0 else 1 end diag_pat_count_ok from diff; - - -/* Test to make sure we got about the same number of patients in the CDM -procedure that we do in i2b2. -*/ -with cdm as ( - select count(distinct patid) qty from procedures - ), -i2b2 as ( - select count(distinct patient_num) qty from ( - select * from i2b2fact - where concept_cd in ( - select concept_cd from "&&i2b2_data_schema".concept_dimension - where concept_path like '\i2b2\Procedures\%' - ) - ) - ), -diff as ( - select - ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct - from cdm cross join i2b2 - ) -select case when diff.pct > 10 then 1/0 else 1 end proc_pat_count_ok from diff; - -/* Test to make sure we have something about patient smoking tobacco use */ -with smokers as ( - select count(*) qty from vital where smoking!='NI' -) -select case when smokers.qty > 0 then 1 else 1/0 end smoker_count_ok from smokers; \ No newline at end of file diff --git a/Oracle/cdm_transform_tests_LOCAL_49608.sql b/Oracle/cdm_transform_tests_LOCAL_49608.sql deleted file mode 100644 index 38140fc..0000000 --- a/Oracle/cdm_transform_tests_LOCAL_49608.sql +++ /dev/null @@ -1,83 +0,0 @@ -/* Test to make sure we got about the same number of patients in the CDM -diagnoses that we do in i2b2. -*/ -with cdm as ( - select count(distinct patid) qty from diagnosis - ), -i2b2 as ( - select count(distinct patient_num) qty from i2b2fact - where concept_cd in ( - select concept_cd from "&&i2b2_data_schema".concept_dimension - where concept_path like '\i2b2\Diagnoses\ICD9\%' - ) - ), -diff as ( - select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct from cdm cross join i2b2 - ) -select case when diff.pct > 10 then 1/0 else 1 end diag_pat_count_ok from diff; - - -/* Test to make sure we got about the same number of patients in the CDM -procedure that we do in i2b2. -*/ -with cdm as ( - select count(distinct patid) qty from procedures - ), -i2b2 as ( - select count(distinct patient_num) qty from ( - select * from i2b2fact - where concept_cd in ( - select concept_cd from "&&i2b2_data_schema".concept_dimension - where concept_path like '\i2b2\Procedures\%' - ) - ) - ), -diff as ( - select - ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct - from cdm cross join i2b2 - ) -select case when diff.pct > 10 then 1/0 else 1 end proc_pat_count_ok from diff; - -/* Test to make sure we have something about patient smoking tobacco use */ -with snums as ( - select smoking cat, count(smoking) qty from vital group by smoking -), -tot as ( - select sum(qty) as cnt from snums -), -calc as ( - select snums.cat, (snums.qty/tot.cnt*100) pct, - case when (snums.qty/tot.cnt*100) > 1 then 1 else 0 end tst - from snums, tot - where snums.cat!='NI' -) -select case when sum(calc.tst) < 3 then 1/0 else 1 end smoking_count_ok from calc; - - -/* Test to make sure we have something about patient general tobacco use */ -with tnums as ( - select tobacco cat, count(tobacco) qty from vital group by tobacco -), -tot as ( - select sum(qty) as cnt from tnums -), -calc as ( - select tnums.cat, (tnums.qty/tot.cnt*100) pct, case when (tnums.qty/tot.cnt*100) > 1 then 1 else 0 end tst from tnums, tot where tnums.cat!='NI' -) -select case when sum(calc.tst) < 3 then 1/0 else 1 end pass from calc; - - -/* Test to make sure we have something about tobacco use types */ -with ttnums as ( - select tobacco_type cat, count(tobacco) qty from vital group by tobacco_type order by cat -), -tot as ( - select sum(qty) as cnt from ttnums -), -calc as ( - select ttnums.cat, (ttnums.qty/tot.cnt*100) pct, case when (ttnums.qty/tot.cnt*100) > 1 then 1 else 0 end tst from ttnums, tot where ttnums.cat!='NI' -) -select case when sum(calc.tst) < 2 then 1/0 else 1 end pass from calc; - - diff --git a/Oracle/cdm_transform_tests_REMOTE_49608.sql b/Oracle/cdm_transform_tests_REMOTE_49608.sql deleted file mode 100644 index 2f21245..0000000 --- a/Oracle/cdm_transform_tests_REMOTE_49608.sql +++ /dev/null @@ -1,139 +0,0 @@ -/* Test to make sure we got about the same number of patients in the CDM -diagnoses that we do in i2b2. -*/ -with cdm as ( - select count(distinct patid) qty from diagnosis - ), -i2b2 as ( - select count(distinct patient_num) qty from i2b2fact - where concept_cd in ( - select concept_cd from "&&i2b2_data_schema".concept_dimension - where concept_path like '\i2b2\Diagnoses\ICD9\%' - ) - ), -diff as ( - select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct from cdm cross join i2b2 - ) -select case when diff.pct > 10 then 1/0 else 1 end diag_pat_count_ok from diff; - - -/* Test to make sure we got about the same number of patients in the CDM -procedure that we do in i2b2. -*/ -with cdm as ( - select count(distinct patid) qty from procedures - ), -i2b2 as ( - select count(distinct patient_num) qty from ( - select * from i2b2fact - where concept_cd in ( - select concept_cd from "&&i2b2_data_schema".concept_dimension - where concept_path like '\i2b2\Procedures\%' - ) - ) - ), -diff as ( - select - ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct - from cdm cross join i2b2 - ) -select case when diff.pct > 10 then 1/0 else 1 end proc_pat_count_ok from diff; - - -/* Make sure we have roughly the same number of Hispanic patients in the CDM and -i2b2. -*/ -with -num_hispanic_cdm as ( - select count(*) qty from demographic where hispanic = 'Y' - ), -num_hispanic_i2b2 as ( - select count(*) qty from "&&i2b2_data_schema".patient_dimension where ethnicity_cd = 'Y' - ), -diff as ( - select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct - from num_hispanic_cdm cdm cross join num_hispanic_i2b2 i2b2 - ) -select case when diff.pct > 10 then 1/0 else 1 end hisp_y_pat_count_ok from diff; - --- TODO: Consider trying to combine the Y and N tests as they are copy/paste -with -num_hispanic_cdm as ( - select count(*) qty from demographic where hispanic = 'N' - ), -num_hispanic_i2b2 as ( - select count(*) qty from "&&i2b2_data_schema".patient_dimension where ethnicity_cd = 'N' - ), -diff as ( - select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct - from num_hispanic_cdm cdm cross join num_hispanic_i2b2 i2b2 - ) -select case when diff.pct > 10 then 1/0 else 1 end hisp_n_pat_count_ok from diff; - --- Make sure we have some diagnosis source information -select case when count(*) < 3 then 1/0 else 1 end a_few_dx_sources from ( - select distinct dx_source from diagnosis - ); - --- Make sure we have valid DX_SOURCE values -select case when count(*) > 0 then 1/0 else 1 end valid_dx_sources from ( - select distinct dx_source from diagnosis where dx_source not in ('AD', 'DI', 'FI', 'IN', 'NI', 'UN', 'OT') - ); - --- Make sure we have a couple principal diagnoses (PDX) -select case when count(*) < 2 then 1/0 else 1 end a_few_pdx_flags from ( - select distinct pdx from diagnosis - ); - --- Make sure we have valid PDX values -select case when count(*) > 0 then 1/0 else 1 end valid_pdx_flags from ( - select distinct pdx from diagnosis where pdx not in ('P', 'S', 'X', 'NI', 'UN', 'OT') - ); - --- Make sure that there are several different enrollment dates -select case when pct_distinct < 5 then 1/0 else 1 end many_enr_dates from ( - with all_enrs as ( - select count(*) qty from enrollment - ), - distinct_date as ( - select count(qty) qty from ( - select distinct enr_start_date qty from enrollment - ) - ) - select round((distinct_date.qty/all_enrs.qty) * 100, 4) pct_distinct - from distinct_date cross join all_enrs - ); - --- Make sure most procedure dates are not null -select case when pct_not_null < 99 then 1/0 else 1 end some_px_dates_not_null from ( - with all_px as ( - select count(*) qty from procedures - ), - not_null as ( - select count(*) qty from procedures where px_date is not null - ) - select round((not_null.qty / all_px.qty) * 100, 4) pct_not_null - from not_null cross join all_px -); - --- Make sure we have some procedure sources (px_source) -select case when count(*) = 0 then 1/0 else 1 end have_px_sources from ( - select distinct px_source from procedures where px_source is not null - ); -/* Test to make sure we have something about patient smoking tobacco use */ -with smokers as ( - select count(*) qty from vital where smoking!='NI' -) -select case when smokers.qty > 0 then 1 else 1/0 end smoker_count_ok from smokers; - --- Make sure we have some encounter types -select case when pct_known < 20 then 1/0 else 1 end some_known_enc_types from ( - with all_enc as ( - select count(*) qty from encounter - ), - known_enc as ( - select count(*) qty from encounter where enc_type is not null and enc_type != 'UN' - ) - select round((known_enc.qty / all_enc.qty) * 100, 4) pct_known - from known_enc cross join all_enc - ); From 624fd25960c116c9993d830a830362b19da22f9c Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 24 Mar 2016 09:12:47 -0500 Subject: [PATCH 106/507] Updated tobacco and tobacco_type transform tests. --- Oracle/cdm_transform_tests.sql | 118 +++++++++++++++++++++++++++------ 1 file changed, 97 insertions(+), 21 deletions(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index af13f8c..9707fbe 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -241,30 +241,106 @@ order by smoking ; -/* Test to make sure we have something about patient general tobacco use */ -with tnums as ( - select tobacco cat, count(tobacco) qty from vital group by tobacco -), -tot as ( - select sum(qty) as cnt from tnums -), -calc as ( - select tnums.cat, (tnums.qty/tot.cnt*100) pct, case when (tnums.qty/tot.cnt*100) > 1 then 1 else 0 end tst from tnums, tot where tnums.cat!='NI' -) -select case when sum(calc.tst) < 3 then 1/0 else 1 end pass from calc; +/* Test to make sure we have something about patient non-smoking tobacco use */ +insert into test_cases (query_name, description, pass + , obs, by_value1, by_value2, record_n, record_pct, distinct_patid_n) + select 'VIT_L3_TOBACCO' query_name + , 'make sure we have something about patient non-smoking tobacco use' description + , case + when tobacco = 'NI' and record_pct < 95 then 1 + when tobacco != 'NI' and distinct_patid_n > 2 then 1 + else 0 + end pass + , rownum obs + , q.* + from ( + with tobacco_terms as ( + select distinct pcori_basecode + from pcornet_vital + where c_fullname like '\PCORI\VITAL\TOBACCO\02\%' + ), + vit as ( + select patid + , tobacco + , case when tobacco = 'NI' then null + when exists ( + select pcori_basecode + from tobacco_terms + where tobacco_terms.pcori_basecode = vital.tobacco) + then null + else 1 + end value_outside + from vital + ), + agg as ( + select count(*) tot from vital) + select tobacco, value_outside + , count(*) + , round(count(*) / tot * 100, 4) record_pct + , count(distinct patid) distinct_patid_n + from vit cross join agg + group by tobacco, value_outside, tot + order by tobacco + ) q + ; /* Test to make sure we have something about tobacco use types */ -with ttnums as ( - select tobacco_type cat, count(tobacco) qty from vital group by tobacco_type order by cat -), -tot as ( - select sum(qty) as cnt from ttnums -), -calc as ( - select ttnums.cat, (ttnums.qty/tot.cnt*100) pct, case when (ttnums.qty/tot.cnt*100) > 1 then 1 else 0 end tst from ttnums, tot where ttnums.cat!='NI' -) -select case when sum(calc.tst) < 2 then 1/0 else 1 end pass from calc; +insert into test_cases (query_name, description, pass + , obs, by_value1, by_value2, record_n, record_pct, distinct_patid_n) + select 'VIT_L3_TOBACCO_TYPE' query_name + , 'make sure we have something about patient tobacco use types' description + , case + when tobacco_type = 'NI' and record_pct < 95 then 1 + when tobacco_type != 'NI' and distinct_patid_n > 2 then 1 + else 0 + end pass + , rownum obs + , q.* + from ( + with tobacco_type_terms as ( + select '01' pcori_basecode from dual + union all + select '02' pcori_basecode from dual + union all + select '03' pcori_basecode from dual + union all + select '04' pcori_basecode from dual + union all + select '05' pcori_basecode from dual + union all + select '06' pcori_basecode from dual + union all + select 'NI' pcori_basecode from dual + union all + select 'UN' pcori_basecode from dual + union all + select 'OT' pcori_basecode from dual + ), + vit as ( + select patid + , tobacco_type + , case when tobacco_type = 'NI' then null + when exists ( + select pcori_basecode + from tobacco_type_terms + where tobacco_type_terms.pcori_basecode = vital.tobacco_type) + then null + else 1 + end value_outside + from vital + ), + agg as ( + select count(*) tot from vital) + select tobacco_type, value_outside + , count(*) + , round(count(*) / tot * 100, 4) record_pct + , count(distinct patid) distinct_patid_n + from vit cross join agg + group by tobacco_type, value_outside, tot + order by tobacco_type + ) q + ; insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) From 4b1d4c8b41c2d63adc1b8e63dcb3b40d236e3642 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Fri, 25 Mar 2016 08:22:32 -0500 Subject: [PATCH 107/507] Oops, use recently added python script to load the local harvest data. --- Oracle/run-i2p-transform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 32aa45d..7861c9e 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -19,7 +19,7 @@ set -e # All i2b2 terms - used for local path mapping #export terms_table= -. ./load_harvest_local.sh +python load_csv.py harvest_local harvest_local.csv harvest_local.ctl pcornet_cdm_user pcornet_cdm . ./load_pcornet_mapping.sh # Run some tests From 2376fea2b1d51f55db8a39d669425c8943fb8c07 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Fri, 25 Mar 2016 11:31:05 -0500 Subject: [PATCH 108/507] Script to back up populated CDM tables by way of rename. --- Oracle/backup_cdm.py | 91 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 Oracle/backup_cdm.py diff --git a/Oracle/backup_cdm.py b/Oracle/backup_cdm.py new file mode 100644 index 0000000..bf5593d --- /dev/null +++ b/Oracle/backup_cdm.py @@ -0,0 +1,91 @@ +''' backup_cdm - back up cdm tables that have at least one row +''' +# Allow --dry run even without cx_Oracle +try: + from cx_Oracle import DatabaseError +except: + pass + +CDM3_TABLES = ['HARVEST', 'DEMOGRAPHIC', 'ENCOUNTER', 'DIAGNOSIS', + 'CONDITION', 'PROCEDURES', 'VITAL', 'ENROLLMENT', + 'LAB_RESULT_CM', 'PRESCRIBING', 'DISPENSING'] + + +def main(get_cursor): + cursor = get_cursor() + + for table in CDM3_TABLES: + if has_rows(cursor, table): + drop_triggers(cursor, table) + drop(cursor, table + '_BAK') + rename(cursor, table, table + '_BAK') + + +def drop_triggers(cursor, table): + cursor.execute("""select trigger_name + from user_triggers + where table_name = '%(table)s'""" % + dict(table=table)) + + for res in cursor.fetchall(): + cursor.execute('drop trigger %(trigger)s' % dict(trigger=res[0])) + + +def drop(cursor, table): + try: + cursor.execute('drop table %(table)s' % dict(table=table)) + except DatabaseError, e: + # Table might not exist + pass + + +def has_rows(cursor, table): + ret = False + try: + cursor.execute('select count(*) from %(table)s' % + dict(table=table)) + if int(cursor.fetchall()[0][0]) > 0: + ret = True + except: + pass + return ret + + +def rename(cursor, table, new_name): + cursor.execute('alter table %(table)s rename to %(new_name)s' % + dict(table=table, new_name=new_name)) + + +class MockCursor(object): + def __init__(self): + self.fetch_count = 0 + + def execute(self, sql): + print 'execute: ' + sql + + def fetchall(self): + r = [(self.fetch_count, )] + self.fetch_count += 1 + print 'fetch returning: ' + str(r) + return r + +if __name__ == '__main__': + def _tcb(): + from os import environ + from sys import argv + + def get_cursor(): + host, port, sid = argv[1:4] + user = environ['pcornet_cdm_user'] + password = environ['pcornet_cdm'] + + if '--dry-run' in argv: + return MockCursor() + else: + import cx_Oracle as cx + conn = cx.connect(user, password, + dsn=cx.makedsn(host, port, sid)) + return conn.cursor() + + main(get_cursor) + _tcb() From 2f116061fe2c04bf164955ea94036800a658fb1e Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Fri, 25 Mar 2016 15:15:48 -0500 Subject: [PATCH 109/507] Populate prescribing table (GPC ticket 496) - Replace PCORNet medication ontology with the HERON medication ontology - add rxcui from mapping at ETL time. - Map HERON order type modifiers to PCORI prescribing modifiers. - Move kludge to add the pcori_cui column from the transform to the ontology mapping code. - Merge provider in to the prescribing table as part of the post-processing. - Test early on that the clarity medication to rxcui mapping exists in the ETL schema --- Oracle/PCORNetLoader_ora.sql | 5 --- Oracle/cdm_postproc.sql | 5 +++ Oracle/pcornet_mapping.csv | 4 +++ Oracle/pcornet_mapping.sql | 64 ++++++++++++++++++++++++++++++++++++ Oracle/run-i2p-transform.sh | 6 ++++ 5 files changed, 79 insertions(+), 5 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 5d4ddcf..f3baadb 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1424,8 +1424,6 @@ end PCORNetHarvest; At compile time, it's complaining about the fact tables don't exist that are created in the function itself. I created them ahead of time - SQL taken from the procedure. - -Also: Error(71,159): PL/SQL: ORA-00904: "MO"."PCORI_CUI": invalid identifier */ whenever sqlerror continue; drop table basis; @@ -1465,9 +1463,6 @@ create table supply( concept_cd varchar2(50 byte) ); -alter table "&&i2b2_meta_schema".pcornet_med add ( - pcori_cui varchar2(1000) -- arbitrary - ); whenever sqlerror exit; create or replace procedure PCORNetPrescribing as diff --git a/Oracle/cdm_postproc.sql b/Oracle/cdm_postproc.sql index 854cfeb..7487bf8 100644 --- a/Oracle/cdm_postproc.sql +++ b/Oracle/cdm_postproc.sql @@ -15,6 +15,11 @@ using encounter e on (p.encounterid = e.encounterid) when matched then update set p.providerid = e.providerid; +merge into prescribing p +using encounter e + on (p.encounterid = e.encounterid) +when matched then update set p.rx_providerid = e.providerid; + /* Currently in HERON, systolic and diastolic blood pressures from flowsheets are reversed. This is fixed in the HERON ETL code (#4026) but we haven't run a production ETL yet. diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 6b9fe57..f76b659 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -1,4 +1,8 @@ pcori_path,local_path +\PCORI_MOD\RX_BASIS\PR\,\Medication\Inpatient\ +\PCORI_MOD\RX_BASIS\PR\,\Medication\Other Orders\ +\PCORI_MOD\RX_BASIS\PR\,\Medication\Outpatient\ +\PCORI_MOD\RX_BASIS\PR\,\Medication\PRN Inpatient Order\ \PCORI_MOD\PDX\P\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\discharge_primary\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\DI\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\AD\,\Diagnosis\\modifier_dx\billing_diagnosis\admit\ diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index e17d591..cbd9597 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -305,3 +305,67 @@ join "&&i2b2_meta_schema".pcornet_enc pe on pe.c_fullname = pm.pcori_path where ht.c_fullname like pm.like_local_path order by c_hlevel ; + + +/* Medications +This section takes the place of PCORI_MEDS_SCHEMA_CHANGE_ora.sql included in the +SCILHS code. +*/ +alter table "&&i2b2_meta_schema".pcornet_med add ( + pcori_cui varchar2(8) + ); +whenever sqlerror exit; + +delete +from "&&i2b2_meta_schema".PCORNET_MED +where c_fullname like '\PCORI\MEDICATION\RXNORM_CUI\%' +and c_fullname not like '\PCORI_MOD\%'; + + +insert into "&&i2b2_meta_schema".PCORNET_MED +with +rxnorm_mapping as ( + /* TODO: Consider whether we want just one rxcui for a clarity medication? + Without picking just one this query results in duplicate c_fullnames and causes + errors in the webclient. + + TODO: Consider changing HERON paths to be RXCUIs or including the RXCUI column + so that we don't have to reach back to the clarity_med_id_to_rxcui map. See + also https://informatics.gpcnetwork.org/trac/Project/ticket/390. + */ + select min(rxcui) rxcui, clarity_med_id + from "&&i2b2_etl_schema". clarity_med_id_to_rxcui@id + group by clarity_med_id + ), +terms_rx as ( + select + cm2rx.rxcui mapped_rxcui, ht.* + from + "&&i2b2_meta_schema"."&&terms_table" ht + left join rxnorm_mapping cm2rx on to_char(cm2rx.clarity_med_id) = replace(ht.c_basecode, 'KUH|MEDICATION_ID:', '') + where c_fullname like '\i2b2\Medications%' + and ht.c_visualattributes not like '%H%' + ) +select + rx.c_hlevel + 1 c_hlevel, + replace(rx.c_fullname, '\i2b2\Medications\', '\PCORI\MEDICATION\RXNORM_CUI\') c_fullname, + rx.c_name, rx.c_synonym_cd, rx.c_visualattributes, + rx.c_totalnum, rx.c_basecode, rx.c_metadataxml, rx.c_facttablecolumn, rx.c_tablename, + rx.c_columnname, rx.c_columndatatype, rx.c_operator, rx.c_dimcode, rx.c_comment, + rx.c_tooltip, rx.m_applied_path, rx.update_date, rx.download_date, rx.import_date, + rx.sourcesystem_cd, rx.valuetype_cd, rx.m_exclusion_cd, rx.c_path, rx.c_symbol, + rx.rxcui pcori_basecode, rx.rxcui pcori_cui, + -- Don't worry about NDCs for now - used for dispensing + null pcori_ndc from ( + select + trx.*, + case + when trx.mapped_rxcui is not null then trx.mapped_rxcui + when trx.c_basecode like 'RXCUI:%' then replace(trx.c_basecode, 'RXCUI:', '') + else null + end rxcui + from terms_rx trx + --order by trx.c_hlevel + ) rx; + +commit; diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 7861c9e..9832eeb 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -15,6 +15,7 @@ set -e #export datamart_name= #export network_id= #export network_name= +# export i2b2_etl_schema= # All i2b2 terms - used for local path mapping #export terms_table= @@ -29,6 +30,7 @@ connect ${pcornet_cdm_user}/${pcornet_cdm} set echo on; define i2b2_data_schema=${i2b2_data_schema} +define i2b2_etl_schema=${i2b2_etl_schema} WHENEVER SQLERROR EXIT SQL.SQLCODE; @@ -45,6 +47,9 @@ select case when qty = 0 then 1/0 else 1 end inout_cd_populated from ( select count(*) qty from "&&i2b2_data_schema".visit_dimension where inout_cd is not null ); +-- Make sure the RXNorm mapping table exists +select rxcui from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id where 1=0; + EOF @@ -78,6 +83,7 @@ WHENEVER SQLERROR EXIT SQL.SQLCODE; define i2b2_data_schema=${i2b2_data_schema} define i2b2_meta_schema=${i2b2_meta_schema} +define i2b2_etl_schema=${i2b2_etl_schema} define datamart_id=${datamart_id} define datamart_name=${datamart_name} define network_id=${network_id} From c49251cc0fdc5cdd956b8f19336976da5c211ed3 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Fri, 25 Mar 2016 16:10:02 -0500 Subject: [PATCH 110/507] Add pcori_ndc column to avoid "too many values" error. - Don't fail if pcori_cui, pcori_ndc columns already exist. - Remove blank line in multi-line comment - sqlplus throws and error because of this. --- Oracle/pcornet_mapping.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index cbd9597..2a3f634 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -311,9 +311,14 @@ order by c_hlevel This section takes the place of PCORI_MEDS_SCHEMA_CHANGE_ora.sql included in the SCILHS code. */ +whenever sqlerror continue; alter table "&&i2b2_meta_schema".pcornet_med add ( pcori_cui varchar2(8) ); + +alter table "&&i2b2_meta_schema".pcornet_med add ( + pcori_ndc varchar2(12) + ); whenever sqlerror exit; delete @@ -328,7 +333,6 @@ rxnorm_mapping as ( /* TODO: Consider whether we want just one rxcui for a clarity medication? Without picking just one this query results in duplicate c_fullnames and causes errors in the webclient. - TODO: Consider changing HERON paths to be RXCUIs or including the RXCUI column so that we don't have to reach back to the clarity_med_id_to_rxcui map. See also https://informatics.gpcnetwork.org/trac/Project/ticket/390. From da68dac759b23755cdc2a4fe8f849ffa5dbfbf5a Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Fri, 25 Mar 2016 18:02:31 -0500 Subject: [PATCH 111/507] Populate RX_BASIS field in the prescribing table (GPC ticket 496) - Map HERON modifiers to PCORI modifiers for RX_BASIS. - Split the pcori_basecode on ':' to get the right basis code in the transform. - Use "like" rather than "=" for matching modifier paths with '%'. --- Oracle/PCORNetLoader_ora.sql | 6 +++-- Oracle/pcornet_mapping.csv | 49 ++++++++++++++++++++++++++++++++++ Oracle/pcornet_mapping.sql | 51 ++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index f3baadb..49091e7 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1536,7 +1536,8 @@ insert into prescribing ( -- ,RAW_RXNORM_CUI ) select distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH:MI'), m.start_date start_date, m.end_date, mo.pcori_cui - ,quantity.nval_num quantity, refills.nval_num refills, supply.nval_num supply, freq.pcori_basecode frequency, basis.pcori_basecode basis + ,quantity.nval_num quantity, refills.nval_num refills, supply.nval_num supply, freq.pcori_basecode frequency, + substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) basis from i2b2fact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode inner join encounter enc on enc.encounterid = m.encounter_Num -- TODO: This join adds several minutes to the load - must be debugged @@ -1561,7 +1562,8 @@ inner join encounter enc on enc.encounterid = m.encounter_Num on m.encounter_num = supply.encounter_num and m.concept_cd = supply.concept_Cd -where (basis.c_fullname is null or basis.c_fullname= '\PCORI_MOD\RX_BASIS\PR\%'); -- jgk 11/2 bugfix: filter for PR, not DI +where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); + end PCORNetPrescribing; / diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index f76b659..596f9cf 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -3,6 +3,55 @@ pcori_path,local_path \PCORI_MOD\RX_BASIS\PR\,\Medication\Other Orders\ \PCORI_MOD\RX_BASIS\PR\,\Medication\Outpatient\ \PCORI_MOD\RX_BASIS\PR\,\Medication\PRN Inpatient Order\ +\PCORI_MOD\RX_BASIS\PR\01\,\Medication\Dispensed\ +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Automatically Held\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Bolus from Syringe\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Bolus.\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Bolus\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Cabinet Pull\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Canceled Entry\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Downtime Given - New Bag\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Downtime Given\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Downtime Not Given\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Due\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\ED-Infusing Upon Admit\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Given - Without Order\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Given - Patient Supplied\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Given - Without Order\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Given by another Provider\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Given\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Held\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Infusion Completed\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Infusion Home with Patient\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\MAR Hold\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\MAR Unhold\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Med Not Available\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Med Not Given - Resch\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Missed\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\NPO\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\New Bag\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\PCA Check/Change\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\PHTN Order Verified\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Patch Applied\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Patch Removed\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Patient/Family Admin\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Paused\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Peak/Trough\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Pending\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Per Protocol\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Piggyback Rate Change\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Push\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Rate Change\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Rate Verify\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Refused\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Restarted\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\See Alternative\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\See ED Trauma Flowsheet\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\See OR/Proc Flowsheet\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Stopped\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Waste\" +"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Wasted\" \PCORI_MOD\PDX\P\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\discharge_primary\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\DI\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\AD\,\Diagnosis\\modifier_dx\billing_diagnosis\admit\ diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 2a3f634..b67d147 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -326,6 +326,13 @@ from "&&i2b2_meta_schema".PCORNET_MED where c_fullname like '\PCORI\MEDICATION\RXNORM_CUI\%' and c_fullname not like '\PCORI_MOD\%'; +update "&&i2b2_meta_schema".PCORNET_MED med +set med.c_visualattributes = 'DAE' where c_fullname in ( + select distinct pm.pcori_path + from pcornet_mapping pm + join "&&i2b2_meta_schema".PCORNET_MED med on med.c_fullname = pm.pcori_path + where med.c_fullname like '\PCORI_MOD\%' + ); insert into "&&i2b2_meta_schema".PCORNET_MED with @@ -372,4 +379,48 @@ select --order by trx.c_hlevel ) rx; +delete +from "&&i2b2_meta_schema".PCORNET_MED where sourcesystem_cd='MAPPING'; + +-- Other diagnosis mappings such as modifiers for PDX, DX_SOURCE. +insert into "&&i2b2_meta_schema".PCORNET_MED +with med_mod_mapping as ( + select distinct pcori_path, i2b2.c_fullname, i2b2.c_basecode, i2b2.c_name + FROM "&&i2b2_meta_schema".pcornet_med + join pcornet_mapping on pcornet_mapping.PCORI_PATH = pcornet_med.c_fullname and pcornet_mapping.local_path is not null + join "&&i2b2_meta_schema"."&&terms_table" i2b2 on i2b2.c_fullname like pcornet_mapping.local_path || '%' + where i2b2.c_basecode is not null + ) +SELECT pcornet_med.C_HLEVEL+1, + pcornet_med.C_FULLNAME || med_mod_mapping.c_basecode || '\' as C_FULLNAME, + med_mod_mapping.c_basecode || ' ' || med_mod_mapping.c_name as C_NAME, + pcornet_med.C_SYNONYM_CD, + pcornet_med.C_VISUALATTRIBUTES, + pcornet_med.C_TOTALNUM, + med_mod_mapping.c_basecode as C_BASECODE, + pcornet_med.C_METADATAXML, + pcornet_med.C_FACTTABLECOLUMN, + pcornet_med.C_TABLENAME, + pcornet_med.C_COLUMNNAME, + pcornet_med.C_COLUMNDATATYPE, + pcornet_med.C_OPERATOR, + pcornet_med.C_FULLNAME || med_mod_mapping.c_basecode || '\' as C_DIMCODE, + pcornet_med.C_COMMENT, + pcornet_med.C_TOOLTIP, + pcornet_med.M_APPLIED_PATH, + pcornet_med.UPDATE_DATE, + pcornet_med.DOWNLOAD_DATE, + pcornet_med.IMPORT_DATE, + 'MAPPING' as SOURCESYSTEM_CD, + pcornet_med.VALUETYPE_CD, + pcornet_med.M_EXCLUSION_CD, + pcornet_med.C_PATH, + pcornet_med.C_SYMBOL, + pcornet_med.PCORI_BASECODE, + null pcori_cui, + null pcori_ndc +from med_mod_mapping +join "&&i2b2_meta_schema".pcornet_med on med_mod_mapping.PCORI_PATH = pcornet_med.c_fullname +; + commit; From 3eee53e9714058b4030858d829ce8b3aeee3573f Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 28 Mar 2016 10:42:09 -0500 Subject: [PATCH 112/507] Use replacement variables for dates limiting patients/visits. - Removed unused i2b2loyalty_patients view - the SCILHS version remains commented out. --- Oracle/PCORNetLoader_ora.sql | 12 ++---------- Oracle/run-i2p-transform.sh | 4 ++++ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 49091e7..a4ec357 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -75,14 +75,14 @@ END; CREATE table i2b2patient_list as select * from ( -select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('01-Jan-2010','dd-mon-rrrr') +select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') ) where ROWNUM<100000000 / create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) / -create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('01-Jan-2010','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE = to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE Date: Mon, 28 Mar 2016 11:38:57 -0500 Subject: [PATCH 113/507] Use variable for the number of months in the past used to calculate enrollment. - The value is adjusted depending on whether or not we're running the transform against date-shifted data. --- Oracle/PCORNetLoader_ora.sql | 2 +- Oracle/run-i2p-transform.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index a4ec357..940c6e4 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1206,7 +1206,7 @@ with pats_delta as ( -- If only one visit, visit_delta_days will be 0 select patient_num, max(start_date) - min(start_date) visit_delta_days from i2b2visit - where start_date > add_months(sysdate, -36) + where start_date > add_months(sysdate, -&&enrollment_months_back) group by patient_num ), enrolled as ( diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 09ab874..d55a49b 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -18,6 +18,7 @@ set -e # export i2b2_etl_schema= # export min_pat_list_date_dd_mon_rrrr= # export min_visit_date_dd_mon_rrrr= +# export enrollment_months_back=36 # All i2b2 terms - used for local path mapping #export terms_table= @@ -93,6 +94,7 @@ define network_name=${network_name} define terms_table=${terms_table} define min_pat_list_date_dd_mon_rrrr=${min_pat_list_date_dd_mon_rrrr} define min_visit_date_dd_mon_rrrr=${min_visit_date_dd_mon_rrrr} +define enrollment_months_back=${enrollment_months_back} -- Local terminology mapping start pcornet_mapping.sql From e94a6dfc2029eb23f2e59bff0412919ebcef5781 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 28 Mar 2016 14:38:12 -0500 Subject: [PATCH 114/507] Replace default SCHILS PCORI_LAB hierarchy with local lab result hierarchy. --- Oracle/pcornet_mapping.sql | 83 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index b67d147..4a7e76c 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -424,3 +424,86 @@ join "&&i2b2_meta_schema".pcornet_med on med_mod_mapping.PCORI_PATH = pcornet_me ; commit; + +/* Replace PCORNet Labs (LOINC) hierarchy with the local hierarchy. +*/ + +-- Create temporary table for mapping LOINC codes to PCORI specimen source values +CREATE TABLE TEMP_LAB_LOINC_MAP (PCORI_BASECODE VARCHAR2(50 BYTE), PCORI_SPECIMEN_SOURCE VARCHAR2(50 BYTE)); +INSERT ALL + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','10839-9') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','12187-1') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','12189-7') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','13969-1') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','14682-9') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','14775-1') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','16255-2') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','17856-6') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','20509-6') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','20569-0') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','21232-4') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','2154-3') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','2157-6') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','2160-0') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','30313-1') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','30350-3') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','30351-1') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','30352-9') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','32673-6') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','33204-9') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','34714-6') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','35203-9') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','38483-4') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','42757-5') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','44784-7') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','4548-4') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','46418-0') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','47213-4') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','48425-3') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','48426-1') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','49136-5') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','49551-5') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','49563-0') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','50756-6') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','55782-7') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','59260-0') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','59826-8') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('PPP','6301-6') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','6597-9') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','6598-7') + INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','718-7') +SELECT * FROM DUAL; + +-- Delete SCHILS Lab hierarchy +delete +from "&&i2b2_meta_schema".PCORNET_LAB +where c_fullname like '\PCORI\LAB_RESULT\%' +; + +-- Replace with local Lab hierarchy + PCORI_BASECODE + PCORI_SPECIMEN_SOURCE +insert into "&&i2b2_meta_schema".PCORNET_LAB +with terms_loinc as ( + select + ht.*, replace(ht.c_basecode, 'LOINC:', '') lc + from + "&&i2b2_meta_schema"."&&terms_table" ht + where c_fullname like '\i2b2\Laboratory Tests\%' and c_basecode like 'LOINC:%' order by c_hlevel + ) +select + tl.c_hlevel, + replace(tl.c_fullname, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT\') c_fullname, + tl.c_name, tl.c_synonym_cd, tl.c_visualattributes, + tl.c_totalnum, tl.c_basecode, tl.c_metadataxml, tl.c_facttablecolumn, tl.c_tablename, + tl.c_columnname, tl.c_columndatatype, tl.c_operator, tl.c_dimcode, tl.c_comment, + tl.c_tooltip, tl.m_applied_path, tl.update_date, tl.download_date, tl.import_date, + tl.sourcesystem_cd, tl.valuetype_cd, tl.m_exclusion_cd, tl.c_path, tl.c_symbol, + tllm.pcori_basecode, tl.lc pcori_specimen_souce +from terms_loinc tl left join temp_lab_loinc_map tllm on tl.lc = tllm.pcori_specimen_source +order by c_hlevel +; + +-- Drop temporary table +DROP TABLE TEMP_LAB_LOINC_MAP; + +commit; + From 1976883beaf51296ae1a8a2dda61a7a4a2ada752 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 28 Mar 2016 16:02:36 -0500 Subject: [PATCH 115/507] Attempting to correct PCORI_LAB path issue which causes tranform to fail. --- Oracle/pcornet_mapping.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 4a7e76c..ea777a2 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -491,7 +491,7 @@ with terms_loinc as ( ) select tl.c_hlevel, - replace(tl.c_fullname, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT\') c_fullname, + replace(tl.c_fullname, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_fullname, tl.c_name, tl.c_synonym_cd, tl.c_visualattributes, tl.c_totalnum, tl.c_basecode, tl.c_metadataxml, tl.c_facttablecolumn, tl.c_tablename, tl.c_columnname, tl.c_columndatatype, tl.c_operator, tl.c_dimcode, tl.c_comment, From 8051c443a8ae865f0e827ca493b9be2b0aa1b13d Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 28 Mar 2016 16:35:00 -0500 Subject: [PATCH 116/507] Forgot update the path while removing the SCHILS paths. --- Oracle/pcornet_mapping.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index ea777a2..9344cf3 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -477,7 +477,7 @@ SELECT * FROM DUAL; -- Delete SCHILS Lab hierarchy delete from "&&i2b2_meta_schema".PCORNET_LAB -where c_fullname like '\PCORI\LAB_RESULT\%' +where c_fullname like '\PCORI\LAB_RESULT_CM\%' ; -- Replace with local Lab hierarchy + PCORI_BASECODE + PCORI_SPECIMEN_SOURCE From c6a90e3bd9889152e4082d0887f08c64bd087c00 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 30 Mar 2016 09:47:18 -0500 Subject: [PATCH 117/507] prune systolic / diastolic swap --- Oracle/cdm_postproc.sql | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Oracle/cdm_postproc.sql b/Oracle/cdm_postproc.sql index 7487bf8..d750775 100644 --- a/Oracle/cdm_postproc.sql +++ b/Oracle/cdm_postproc.sql @@ -20,12 +20,6 @@ using encounter e on (p.encounterid = e.encounterid) when matched then update set p.rx_providerid = e.providerid; -/* Currently in HERON, systolic and diastolic blood pressures from flowsheets -are reversed. This is fixed in the HERON ETL code (#4026) but we haven't run a -production ETL yet. -*/ -update vital v set v.systolic = v.diastolic, v.diastolic = v.systolic; - /* Currently in HERON, we have hight in cm and weight in kg - the CDM wants height in inches and weight in pounds. */ update vital v set v.ht = v.ht / 2.54; From 44edc731d784ab0b49e1ccbf43a8eeecb46d4bf6 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 30 Mar 2016 10:20:07 -0500 Subject: [PATCH 118/507] Populate death table. --- Oracle/cdm_postproc.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Oracle/cdm_postproc.sql b/Oracle/cdm_postproc.sql index d750775..9e90267 100644 --- a/Oracle/cdm_postproc.sql +++ b/Oracle/cdm_postproc.sql @@ -24,3 +24,15 @@ when matched then update set p.rx_providerid = e.providerid; height in inches and weight in pounds. */ update vital v set v.ht = v.ht / 2.54; update vital v set v.wt = v.wt * 2.20462; + +/* Populate death table. Eventually, we expect this to be added to the upstream +transform code. +Ref: https://github.com/SCILHS/i2p-transform/issues/3 +*/ +insert into death +select + pd.patient_num, pd.death_date, 'N' death_date_impute, + 'UN' death_source, -- TODO: We have SSMDF and EMR sources at least + 'E' death_match_confidence +from "&&i2b2_data_schema".patient_dimension pd +where pd.vital_status_cd = 'y'; From 16d26bb8f9126a8e51dba01b8eaec918f310b1f6 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 30 Mar 2016 16:27:23 -0500 Subject: [PATCH 119/507] move Race '@' from OT to NI in PCORNET_DEMO --- Oracle/pcornet_ontology_updates.csv | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Oracle/pcornet_ontology_updates.csv b/Oracle/pcornet_ontology_updates.csv index 9544604..4c09710 100644 --- a/Oracle/pcornet_ontology_updates.csv +++ b/Oracle/pcornet_ontology_updates.csv @@ -5,3 +5,5 @@ 3,"\PCORI\DEMOGRAPHIC\HISPANIC\OT\","Other","N","LAE",,"ETHNICITY:OT",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'OT'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2014-06-02 19:44:16","2014-06-02 19:44:16","2014-06-02 19:44:16","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","OT","OT" 3,"\PCORI\DEMOGRAPHIC\HISPANIC\UN\","Unknown","N","LAE",,"ETHNICITY:UN",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'UN','u','unk'",,"Unknown: A data field is present in the source system, but the source value explicitly denotes an unknown value.","@","2014-06-02 19:43:08","2014-06-02 19:43:08","2014-06-02 19:43:08","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","UN","UN" 3,"\PCORI\DEMOGRAPHIC\HISPANIC\Y\","Hispanic","N","LAE",,"ETHNICITY:HISPANIC",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'Y','hib','his/black','his/white','hiw','hispanic'",,"A person of Cuban, Mexican, Puerto Rican, South or Central American, or other Spanish culture or origin, regardless of race. \ Hisptanic","@","2014-05-09 15:21:33","2014-05-09 11:11:12","2014-05-09 11:11:12","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","Y","Y" +3,"\PCORI\DEMOGRAPHIC\RACE\NI\","No information","N","LAE",,"RACE:NI",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","N","IS","@'",,"No information: A data field is present in the source system, but the source value is null or blank.","@","2014/06/02","2014/06/02","2014/06/02","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","NI","NI" +3,"\PCORI\DEMOGRAPHIC\RACE\OT\","Other","N","LAE",,"RACE:OT",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","T","IN","OT','o','d','deferred'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2014/06/02","2014/06/02","2014/06/02","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","OT","OT" From 783d0b6cb67d728850b4f2e8080d89ba9512bc75 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 30 Mar 2016 16:55:04 -0500 Subject: [PATCH 120/507] fix dates in race updates --- Oracle/pcornet_ontology_updates.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/pcornet_ontology_updates.csv b/Oracle/pcornet_ontology_updates.csv index 4c09710..494cd06 100644 --- a/Oracle/pcornet_ontology_updates.csv +++ b/Oracle/pcornet_ontology_updates.csv @@ -5,5 +5,5 @@ 3,"\PCORI\DEMOGRAPHIC\HISPANIC\OT\","Other","N","LAE",,"ETHNICITY:OT",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'OT'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2014-06-02 19:44:16","2014-06-02 19:44:16","2014-06-02 19:44:16","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","OT","OT" 3,"\PCORI\DEMOGRAPHIC\HISPANIC\UN\","Unknown","N","LAE",,"ETHNICITY:UN",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'UN','u','unk'",,"Unknown: A data field is present in the source system, but the source value explicitly denotes an unknown value.","@","2014-06-02 19:43:08","2014-06-02 19:43:08","2014-06-02 19:43:08","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","UN","UN" 3,"\PCORI\DEMOGRAPHIC\HISPANIC\Y\","Hispanic","N","LAE",,"ETHNICITY:HISPANIC",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'Y','hib','his/black','his/white','hiw','hispanic'",,"A person of Cuban, Mexican, Puerto Rican, South or Central American, or other Spanish culture or origin, regardless of race. \ Hisptanic","@","2014-05-09 15:21:33","2014-05-09 11:11:12","2014-05-09 11:11:12","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","Y","Y" -3,"\PCORI\DEMOGRAPHIC\RACE\NI\","No information","N","LAE",,"RACE:NI",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","N","IS","@'",,"No information: A data field is present in the source system, but the source value is null or blank.","@","2014/06/02","2014/06/02","2014/06/02","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","NI","NI" -3,"\PCORI\DEMOGRAPHIC\RACE\OT\","Other","N","LAE",,"RACE:OT",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","T","IN","OT','o','d','deferred'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2014/06/02","2014/06/02","2014/06/02","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","OT","OT" +3,"\PCORI\DEMOGRAPHIC\RACE\NI\","No information","N","LAE",,"RACE:NI",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","N","IS","@'",,"No information: A data field is present in the source system, but the source value is null or blank.","@","2016-03-30 16:48:29","2016-03-30 16:48:29","2016-03-30 16:48:29","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","NI","NI" +3,"\PCORI\DEMOGRAPHIC\RACE\OT\","Other","N","LAE",,"RACE:OT",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","T","IN","OT','o','d','deferred'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2016-03-30 16:48:29","2016-03-30 16:48:29","2016-03-30 16:48:29","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","OT","OT" From 2a0319f643e11b8719bd72929da14cef97fccbba Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 30 Mar 2016 17:28:00 -0500 Subject: [PATCH 121/507] oops: missing ' around '@' --- Oracle/pcornet_ontology_updates.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_ontology_updates.csv b/Oracle/pcornet_ontology_updates.csv index 494cd06..aa0294c 100644 --- a/Oracle/pcornet_ontology_updates.csv +++ b/Oracle/pcornet_ontology_updates.csv @@ -5,5 +5,5 @@ 3,"\PCORI\DEMOGRAPHIC\HISPANIC\OT\","Other","N","LAE",,"ETHNICITY:OT",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'OT'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2014-06-02 19:44:16","2014-06-02 19:44:16","2014-06-02 19:44:16","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","OT","OT" 3,"\PCORI\DEMOGRAPHIC\HISPANIC\UN\","Unknown","N","LAE",,"ETHNICITY:UN",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'UN','u','unk'",,"Unknown: A data field is present in the source system, but the source value explicitly denotes an unknown value.","@","2014-06-02 19:43:08","2014-06-02 19:43:08","2014-06-02 19:43:08","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","UN","UN" 3,"\PCORI\DEMOGRAPHIC\HISPANIC\Y\","Hispanic","N","LAE",,"ETHNICITY:HISPANIC",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'Y','hib','his/black','his/white','hiw','hispanic'",,"A person of Cuban, Mexican, Puerto Rican, South or Central American, or other Spanish culture or origin, regardless of race. \ Hisptanic","@","2014-05-09 15:21:33","2014-05-09 11:11:12","2014-05-09 11:11:12","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","Y","Y" -3,"\PCORI\DEMOGRAPHIC\RACE\NI\","No information","N","LAE",,"RACE:NI",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","N","IS","@'",,"No information: A data field is present in the source system, but the source value is null or blank.","@","2016-03-30 16:48:29","2016-03-30 16:48:29","2016-03-30 16:48:29","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","NI","NI" +3,"\PCORI\DEMOGRAPHIC\RACE\NI\","No information","N","LAE",,"RACE:NI",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","N","IS","'@'",,"No information: A data field is present in the source system, but the source value is null or blank.","@","2016-03-30 16:48:29","2016-03-30 16:48:29","2016-03-30 16:48:29","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","NI","NI" 3,"\PCORI\DEMOGRAPHIC\RACE\OT\","Other","N","LAE",,"RACE:OT",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","T","IN","OT','o','d','deferred'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2016-03-30 16:48:29","2016-03-30 16:48:29","2016-03-30 16:48:29","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","OT","OT" From d4aed3c0a494706a189867a181149ce1095cbf15 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 30 Mar 2016 17:36:40 -0500 Subject: [PATCH 122/507] yet another race dimcode fix --- Oracle/pcornet_ontology_updates.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_ontology_updates.csv b/Oracle/pcornet_ontology_updates.csv index aa0294c..b5c10e2 100644 --- a/Oracle/pcornet_ontology_updates.csv +++ b/Oracle/pcornet_ontology_updates.csv @@ -6,4 +6,4 @@ 3,"\PCORI\DEMOGRAPHIC\HISPANIC\UN\","Unknown","N","LAE",,"ETHNICITY:UN",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'UN','u','unk'",,"Unknown: A data field is present in the source system, but the source value explicitly denotes an unknown value.","@","2014-06-02 19:43:08","2014-06-02 19:43:08","2014-06-02 19:43:08","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","UN","UN" 3,"\PCORI\DEMOGRAPHIC\HISPANIC\Y\","Hispanic","N","LAE",,"ETHNICITY:HISPANIC",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'Y','hib','his/black','his/white','hiw','hispanic'",,"A person of Cuban, Mexican, Puerto Rican, South or Central American, or other Spanish culture or origin, regardless of race. \ Hisptanic","@","2014-05-09 15:21:33","2014-05-09 11:11:12","2014-05-09 11:11:12","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","Y","Y" 3,"\PCORI\DEMOGRAPHIC\RACE\NI\","No information","N","LAE",,"RACE:NI",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","N","IS","'@'",,"No information: A data field is present in the source system, but the source value is null or blank.","@","2016-03-30 16:48:29","2016-03-30 16:48:29","2016-03-30 16:48:29","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","NI","NI" -3,"\PCORI\DEMOGRAPHIC\RACE\OT\","Other","N","LAE",,"RACE:OT",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","T","IN","OT','o','d','deferred'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2016-03-30 16:48:29","2016-03-30 16:48:29","2016-03-30 16:48:29","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","OT","OT" +3,"\PCORI\DEMOGRAPHIC\RACE\OT\","Other","N","LAE",,"RACE:OT",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","T","IN","'OT','o','d','deferred'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2016-03-30 16:48:29","2016-03-30 16:48:29","2016-03-30 16:48:29","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","OT","OT" From f894d130ba1b1e7eb523c17c02f00e9189cf413e Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 30 Mar 2016 17:45:55 -0500 Subject: [PATCH 123/507] Build the harvest table last since it needs the other tables built first. --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 940c6e4..1750f37 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1716,7 +1716,6 @@ end pcornetReport; create or replace procedure pcornetloader as begin ---pcornetclear; -PCORNetHarvest; PCORNetDemographic; PCORNetEncounter; PCORNetDiagnosis; @@ -1748,6 +1747,7 @@ http://listserv.kumc.edu/pipermail/gpc-dev/attachments/20160223/8d79fa70/attachm > what we have in our i2b2 */ --PCORNetDispensing; +PCORNetHarvest; end pcornetloader; / From 21924791e807fda952d299b03176f5052e75b09a Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 25 Apr 2016 09:43:43 -0500 Subject: [PATCH 124/507] Add relevent nodes from local i2b2 lab hierarchy to SCHILS Lab hierarchy. --- Oracle/pcornet_mapping.sql | 115 +++++++++++++------------------------ 1 file changed, 40 insertions(+), 75 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 9344cf3..7521b5a 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -425,85 +425,50 @@ join "&&i2b2_meta_schema".pcornet_med on med_mod_mapping.PCORI_PATH = pcornet_me commit; -/* Replace PCORNet Labs (LOINC) hierarchy with the local hierarchy. +/* Add relevent nodes from local i2b2 lab hierarchy to PCORNet Labs hierarchy. */ --- Create temporary table for mapping LOINC codes to PCORI specimen source values -CREATE TABLE TEMP_LAB_LOINC_MAP (PCORI_BASECODE VARCHAR2(50 BYTE), PCORI_SPECIMEN_SOURCE VARCHAR2(50 BYTE)); -INSERT ALL - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','10839-9') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','12187-1') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','12189-7') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','13969-1') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','14682-9') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','14775-1') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','16255-2') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','17856-6') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','20509-6') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','20569-0') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','21232-4') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','2154-3') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','2157-6') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','2160-0') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','30313-1') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','30350-3') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','30351-1') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','30352-9') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','32673-6') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','33204-9') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','34714-6') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','35203-9') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','38483-4') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','42757-5') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','44784-7') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','4548-4') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','46418-0') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','47213-4') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','48425-3') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','48426-1') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','49136-5') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','49551-5') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','49563-0') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','50756-6') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','55782-7') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','59260-0') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','59826-8') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('PPP','6301-6') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','6597-9') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('SERUM, PLASMA, or SR_PLS','6598-7') - INTO TEMP_LAB_LOINC_MAP (PCORI_BASECODE, PCORI_SPECIMEN_SOURCE) VALUES ('BLOOD','718-7') -SELECT * FROM DUAL; - --- Delete SCHILS Lab hierarchy -delete -from "&&i2b2_meta_schema".PCORNET_LAB -where c_fullname like '\PCORI\LAB_RESULT_CM\%' -; - --- Replace with local Lab hierarchy + PCORI_BASECODE + PCORI_SPECIMEN_SOURCE insert into "&&i2b2_meta_schema".PCORNET_LAB -with terms_loinc as ( - select - ht.*, replace(ht.c_basecode, 'LOINC:', '') lc - from - "&&i2b2_meta_schema"."&&terms_table" ht - where c_fullname like '\i2b2\Laboratory Tests\%' and c_basecode like 'LOINC:%' order by c_hlevel - ) -select - tl.c_hlevel, - replace(tl.c_fullname, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_fullname, - tl.c_name, tl.c_synonym_cd, tl.c_visualattributes, - tl.c_totalnum, tl.c_basecode, tl.c_metadataxml, tl.c_facttablecolumn, tl.c_tablename, - tl.c_columnname, tl.c_columndatatype, tl.c_operator, tl.c_dimcode, tl.c_comment, - tl.c_tooltip, tl.m_applied_path, tl.update_date, tl.download_date, tl.import_date, - tl.sourcesystem_cd, tl.valuetype_cd, tl.m_exclusion_cd, tl.c_path, tl.c_symbol, - tllm.pcori_basecode, tl.lc pcori_specimen_souce -from terms_loinc tl left join temp_lab_loinc_map tllm on tl.lc = tllm.pcori_specimen_source -order by c_hlevel +with lab_map as ( + select distinct lab.c_hlevel, lab.c_path, lab.pcori_basecode, trim(CHR(13) from lab.pcori_specimen_source) as pcori_specimen_source + from "&&i2b2_meta_schema".pcornet_lab lab + inner JOIN "&&i2b2_meta_schema".pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname + inner join pmn_labnormal norm on ont_parent.c_basecode=norm.LAB_NAME + where lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' +) +select +lm.C_HLEVEL, +concat(lm.c_path, substr(regexp_substr(lt.c_fullname, '\\[^\\]+\\$'), 2, length(regexp_substr(lt.c_fullname, '\\[^\\]+\\$')))) as c_fullname, +lt.C_NAME, +lt.C_SYNONYM_CD, +lt.C_VISUALATTRIBUTES, +lt.C_TOTALNUM, +lt.C_BASECODE, +lt.C_METADATAXML, +lt.C_FACTTABLECOLUMN, +lt.C_TABLENAME, +lt.C_COLUMNNAME, +lt.C_COLUMNDATATYPE, +lt.C_OPERATOR, +lt.C_DIMCODE, +lt.C_COMMENT, +lt.C_TOOLTIP, +lt.M_APPLIED_PATH, +lt.UPDATE_DATE, +lt.DOWNLOAD_DATE, +lt.IMPORT_DATE, +lt.SOURCESYSTEM_CD, +lt.VALUETYPE_CD, +lt.M_EXCLUSION_CD, +lm.C_PATH, +lt.C_SYMBOL, +lt.TERM_ID, +lm.PCORI_BASECODE, +lm.pcori_specimen_source +from "&&i2b2_meta_schema"."&&terms_table" lt, lab_map lm +where lm.pcori_specimen_source=replace(lt.c_basecode, 'LOINC:', '') +and lt.c_fullname like '\i2b2\Laboratory Tests\%' and lt.c_basecode like 'LOINC:%' ; --- Drop temporary table -DROP TABLE TEMP_LAB_LOINC_MAP; - commit; From ab172f101c870818741a47cf03285d483db97150 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 26 Apr 2016 14:40:56 -0500 Subject: [PATCH 125/507] Removed local column reference. --- Oracle/pcornet_mapping.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 7521b5a..87d12f8 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -462,7 +462,6 @@ lt.VALUETYPE_CD, lt.M_EXCLUSION_CD, lm.C_PATH, lt.C_SYMBOL, -lt.TERM_ID, lm.PCORI_BASECODE, lm.pcori_specimen_source from "&&i2b2_meta_schema"."&&terms_table" lt, lab_map lm From aa9dd537808f71e185ddf87097f872684fde8fb8 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 27 Apr 2016 08:59:09 -0500 Subject: [PATCH 126/507] Moved static PMN_LabNormal creation out of PCORNETLoader.sql for use in mapping. --- Oracle/PCORNetLoader_ora.sql | 60 ------------------------------------ Oracle/pmn_labnormal.csv | 13 ++++++++ Oracle/run-i2p-transform.sh | 1 + 3 files changed, 14 insertions(+), 60 deletions(-) create mode 100644 Oracle/pmn_labnormal.csv diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 1750f37..ac66ea8 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -415,66 +415,6 @@ begin end; / -BEGIN -PMN_DROPSQL('DROP TABLE PMN_LabNormal'); -END; -/ -CREATE TABLE PMN_LabNormal ( - LAB_NAME varchar(150) NULL, - NORM_RANGE_LOW varchar(10) NULL, - NORM_MODIFIER_LOW varchar(2) NULL, - NORM_RANGE_HIGH varchar(10) NULL, - NORM_MODIFIER_HIGH varchar(2) NULL - ) -/ - -INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANGE_HIGH, NORM_MODIFIER_HIGH) - VALUES('LAB_NAME:LDL', '0', 'GE', '165', 'LE') -/ - -INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANGE_HIGH, NORM_MODIFIER_HIGH) - VALUES('LAB_NAME:A1C', '', 'NI', '', 'NI') -/ - -INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANGE_HIGH, NORM_MODIFIER_HIGH) - VALUES('LAB_NAME:CK', '50', 'GE', '236', 'LE') -/ - -INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANGE_HIGH, NORM_MODIFIER_HIGH) - VALUES('LAB_NAME:CK_MB', '', 'NI', '', 'NI') -/ - -INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANGE_HIGH, NORM_MODIFIER_HIGH) - VALUES('LAB_NAME:CK_MBI', '', 'NI', '', 'NI') -/ - -INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANGE_HIGH, NORM_MODIFIER_HIGH) - VALUES('LAB_NAME:CREATININE', '0', 'GE', '1.6', 'LE') -/ - -INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANGE_HIGH, NORM_MODIFIER_HIGH) - VALUES('LAB_NAME:CREATININE', '0', 'GE', '1.6', 'LE') -/ - -INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANGE_HIGH, NORM_MODIFIER_HIGH) - VALUES('LAB_NAME:HGB', '12', 'GE', '17.5', 'LE') -/ - -INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANGE_HIGH, NORM_MODIFIER_HIGH) - VALUES('LAB_NAME:INR', '0.8', 'GE', '1.3', 'LE') -/ - -INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANGE_HIGH, NORM_MODIFIER_HIGH) - VALUES('LAB_NAME:TROP_I', '0', 'GE', '0.49', 'LE') -/ - -INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANGE_HIGH, NORM_MODIFIER_HIGH) - VALUES('LAB_NAME:TROP_T_QL', '', 'NI', '', 'NI') -/ - -INSERT INTO PMN_LabNormal(LAB_NAME, NORM_RANGE_LOW, NORM_MODIFIER_LOW, NORM_RANGE_HIGH, NORM_MODIFIER_HIGH) - VALUES('LAB_NAME:TROP_T_QN', '0', 'GE', '0.09', 'LE') -/ BEGIN PMN_DROPSQL('DROP TABLE death'); diff --git a/Oracle/pmn_labnormal.csv b/Oracle/pmn_labnormal.csv new file mode 100644 index 0000000..6d6277a --- /dev/null +++ b/Oracle/pmn_labnormal.csv @@ -0,0 +1,13 @@ +LAB_NAME,NORM_RANGE_LOW,NORM_MODIFIER_LOW,NORM_RANGE_HIGH,NORM_MODIFIER_HIGH +LAB_NAME:LDL,0,GE,165,LE +LAB_NAME:A1C,,NI,,NI +LAB_NAME:CK,50,GE,236,LE +LAB_NAME:CK_MB,,NI,,NI +LAB_NAME:CK_MBI,,NI,,NI +LAB_NAME:CREATININE,0,GE,1.6,LE +LAB_NAME:CREATININE,0,GE,1.6,LE +LAB_NAME:HGB,12,GE,17.5,LE +LAB_NAME:INR,0.8,GE,1.3,LE +LAB_NAME:TROP_I,0,GE,0.49,LE +LAB_NAME:TROP_T_QL,,NI,,NI +LAB_NAME:TROP_T_QN,0,GE,0.09,LE diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index d55a49b..a740256 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -24,6 +24,7 @@ set -e #export terms_table= python load_csv.py harvest_local harvest_local.csv harvest_local.ctl pcornet_cdm_user pcornet_cdm +python load_csv.py PMN_LabNormal pmn_labnormal.csv pmn_labnormal.ctl pcornet_cdm_user pcornet_cdm . ./load_pcornet_mapping.sh # Run some tests From baa708ff232cd58675873f83c99d4da0f5a91d36 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 2 May 2016 14:20:29 -0500 Subject: [PATCH 127/507] 2nd attempt at hanging the correct local nodes under SCHILS lab nodes. --- Oracle/pcornet_mapping.sql | 108 ++++++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 37 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 87d12f8..295a1c2 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -430,44 +430,78 @@ commit; insert into "&&i2b2_meta_schema".PCORNET_LAB with lab_map as ( - select distinct lab.c_hlevel, lab.c_path, lab.pcori_basecode, trim(CHR(13) from lab.pcori_specimen_source) as pcori_specimen_source - from "&&i2b2_meta_schema".pcornet_lab lab - inner JOIN "&&i2b2_meta_schema".pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname - inner join pmn_labnormal norm on ont_parent.c_basecode=norm.LAB_NAME - where lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' + select distinct lab.c_hlevel, lab.c_path, lab.pcori_basecode, trim(CHR(13) from lab.pcori_specimen_source) as pcori_specimen_source + from pcornet_lab lab + inner JOIN pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname + inner join pmn_labnormal norm on ont_parent.c_basecode=norm.LAB_NAME + where lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' +), +local_loinc_terms as ( + select + lm.C_HLEVEL, + lt.C_FULLNAME, + lt.C_NAME, + lt.C_SYNONYM_CD, + lt.C_VISUALATTRIBUTES, + lt.C_TOTALNUM, + lt.C_BASECODE, + lt.C_METADATAXML, + lt.C_FACTTABLECOLUMN, + lt.C_TABLENAME, + lt.C_COLUMNNAME, + lt.C_COLUMNDATATYPE, + lt.C_OPERATOR, + lt.C_DIMCODE, + lt.C_COMMENT, + lt.C_TOOLTIP, + lt.M_APPLIED_PATH, + lt.UPDATE_DATE, + lt.DOWNLOAD_DATE, + lt.IMPORT_DATE, + lt.SOURCESYSTEM_CD, + lt.VALUETYPE_CD, + lt.M_EXCLUSION_CD, + lm.C_PATH, + lt.C_SYMBOL, + lt.TERM_ID, + lm.PCORI_BASECODE, + lm.pcori_specimen_source + from "&&i2b2_meta_schema"."&&terms_table" lt, lab_map lm + where lm.pcori_specimen_source=replace(lt.c_basecode, 'LOINC:', '') + and lt.c_fullname like '\i2b2\Laboratory Tests\%' and lt.c_basecode like 'LOINC:%' ) -select -lm.C_HLEVEL, -concat(lm.c_path, substr(regexp_substr(lt.c_fullname, '\\[^\\]+\\$'), 2, length(regexp_substr(lt.c_fullname, '\\[^\\]+\\$')))) as c_fullname, -lt.C_NAME, -lt.C_SYNONYM_CD, -lt.C_VISUALATTRIBUTES, -lt.C_TOTALNUM, -lt.C_BASECODE, -lt.C_METADATAXML, -lt.C_FACTTABLECOLUMN, -lt.C_TABLENAME, -lt.C_COLUMNNAME, -lt.C_COLUMNDATATYPE, -lt.C_OPERATOR, -lt.C_DIMCODE, -lt.C_COMMENT, -lt.C_TOOLTIP, -lt.M_APPLIED_PATH, -lt.UPDATE_DATE, -lt.DOWNLOAD_DATE, -lt.IMPORT_DATE, -lt.SOURCESYSTEM_CD, -lt.VALUETYPE_CD, -lt.M_EXCLUSION_CD, -lm.C_PATH, -lt.C_SYMBOL, -lm.PCORI_BASECODE, -lm.pcori_specimen_source -from "&&i2b2_meta_schema"."&&terms_table" lt, lab_map lm -where lm.pcori_specimen_source=replace(lt.c_basecode, 'LOINC:', '') -and lt.c_fullname like '\i2b2\Laboratory Tests\%' and lt.c_basecode like 'LOINC:%' -; +select + llt.C_HLEVEL, + concat(llt.c_path, substr(regexp_substr(lt.c_fullname, '\\[^\\]+\\$'), 2, length(regexp_substr(lt.c_fullname, '\\[^\\]+\\$')))) as c_fullname, + lt.C_NAME, + lt.C_SYNONYM_CD, + lt.C_VISUALATTRIBUTES, + lt.C_TOTALNUM, + lt.C_BASECODE, + lt.C_METADATAXML, + lt.C_FACTTABLECOLUMN, + lt.C_TABLENAME, + lt.C_COLUMNNAME, + lt.C_COLUMNDATATYPE, + lt.C_OPERATOR, + lt.C_DIMCODE, + lt.C_COMMENT, + lt.C_TOOLTIP, + lt.M_APPLIED_PATH, + lt.UPDATE_DATE, + lt.DOWNLOAD_DATE, + lt.IMPORT_DATE, + 'MAPPING', + lt.VALUETYPE_CD, + lt.M_EXCLUSION_CD, + llt.C_PATH, + lt.C_SYMBOL, + lt.TERM_ID, + llt.PCORI_BASECODE, + llt.pcori_specimen_source +from "&&i2b2_meta_schema"."&&terms_table" lt, local_loinc_terms llt +where lt.c_fullname like llt.c_fullname||'%\' +; commit; From 227e2d1f8e6e89bbd575f268ef2e7fbb8062e321 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 4 May 2016 09:33:46 -0500 Subject: [PATCH 128/507] Cleaned up lab mapping SQL. --- Oracle/pcornet_mapping.sql | 103 +++++++++++++------------------------ 1 file changed, 37 insertions(+), 66 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 295a1c2..13dca3a 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -430,78 +430,49 @@ commit; insert into "&&i2b2_meta_schema".PCORNET_LAB with lab_map as ( - select distinct lab.c_hlevel, lab.c_path, lab.pcori_basecode, trim(CHR(13) from lab.pcori_specimen_source) as pcori_specimen_source - from pcornet_lab lab - inner JOIN pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname - inner join pmn_labnormal norm on ont_parent.c_basecode=norm.LAB_NAME - where lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' + select distinct lab.c_hlevel, lab.c_path, lab.pcori_basecode, trim(CHR(13) from lab.pcori_specimen_source) as pcori_specimen_source + from pcornet_lab lab + inner JOIN pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname + inner join pmn_labnormal norm on ont_parent.c_basecode=norm.LAB_NAME + where lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' ), local_loinc_terms as ( - select - lm.C_HLEVEL, - lt.C_FULLNAME, - lt.C_NAME, - lt.C_SYNONYM_CD, - lt.C_VISUALATTRIBUTES, - lt.C_TOTALNUM, - lt.C_BASECODE, - lt.C_METADATAXML, - lt.C_FACTTABLECOLUMN, - lt.C_TABLENAME, - lt.C_COLUMNNAME, - lt.C_COLUMNDATATYPE, - lt.C_OPERATOR, - lt.C_DIMCODE, - lt.C_COMMENT, - lt.C_TOOLTIP, - lt.M_APPLIED_PATH, - lt.UPDATE_DATE, - lt.DOWNLOAD_DATE, - lt.IMPORT_DATE, - lt.SOURCESYSTEM_CD, - lt.VALUETYPE_CD, - lt.M_EXCLUSION_CD, - lm.C_PATH, - lt.C_SYMBOL, - lt.TERM_ID, - lm.PCORI_BASECODE, - lm.pcori_specimen_source - from "&&i2b2_meta_schema"."&&terms_table" lt, lab_map lm - where lm.pcori_specimen_source=replace(lt.c_basecode, 'LOINC:', '') - and lt.c_fullname like '\i2b2\Laboratory Tests\%' and lt.c_basecode like 'LOINC:%' + select lm.C_HLEVEL, lt.C_FULLNAME, lm.C_PATH, lm.PCORI_BASECODE, lm.pcori_specimen_source + from "&&i2b2_meta_schema"."&&terms_table" lt, lab_map lm + where lm.pcori_specimen_source=replace(lt.c_basecode, 'LOINC:', '') + and lt.c_fullname like '\i2b2\Laboratory Tests\%' and lt.c_basecode like 'LOINC:%' ) select - llt.C_HLEVEL, - concat(llt.c_path, substr(regexp_substr(lt.c_fullname, '\\[^\\]+\\$'), 2, length(regexp_substr(lt.c_fullname, '\\[^\\]+\\$')))) as c_fullname, - lt.C_NAME, - lt.C_SYNONYM_CD, - lt.C_VISUALATTRIBUTES, - lt.C_TOTALNUM, - lt.C_BASECODE, - lt.C_METADATAXML, - lt.C_FACTTABLECOLUMN, - lt.C_TABLENAME, - lt.C_COLUMNNAME, - lt.C_COLUMNDATATYPE, - lt.C_OPERATOR, - lt.C_DIMCODE, - lt.C_COMMENT, - lt.C_TOOLTIP, - lt.M_APPLIED_PATH, - lt.UPDATE_DATE, - lt.DOWNLOAD_DATE, - lt.IMPORT_DATE, - 'MAPPING', - lt.VALUETYPE_CD, - lt.M_EXCLUSION_CD, - llt.C_PATH, - lt.C_SYMBOL, - lt.TERM_ID, - llt.PCORI_BASECODE, - llt.pcori_specimen_source + llt.C_HLEVEL, + concat(llt.c_path, substr(regexp_substr(lt.c_fullname, '\\[^\\]+\\$'), 2, length(regexp_substr(lt.c_fullname, '\\[^\\]+\\$')))) as c_fullname, + lt.C_NAME, + lt.C_SYNONYM_CD, + lt.C_VISUALATTRIBUTES, + lt.C_TOTALNUM, + lt.C_BASECODE, + lt.C_METADATAXML, + lt.C_FACTTABLECOLUMN, + lt.C_TABLENAME, + lt.C_COLUMNNAME, + lt.C_COLUMNDATATYPE, + lt.C_OPERATOR, + lt.C_DIMCODE, + lt.C_COMMENT, + lt.C_TOOLTIP, + lt.M_APPLIED_PATH, + lt.UPDATE_DATE, + lt.DOWNLOAD_DATE, + lt.IMPORT_DATE, + 'MAPPING', + lt.VALUETYPE_CD, + lt.M_EXCLUSION_CD, + llt.C_PATH, + lt.C_SYMBOL, + llt.PCORI_BASECODE, + llt.pcori_specimen_source from "&&i2b2_meta_schema"."&&terms_table" lt, local_loinc_terms llt where lt.c_fullname like llt.c_fullname||'%\' -; +; commit; From d4441ddeab3861652fdd846399c5f57c15c7b7db Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 4 May 2016 14:55:56 -0500 Subject: [PATCH 129/507] PMN_LABNORMAL was being dropped before it could be used. --- Oracle/run-i2p-transform.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index a740256..77c08e7 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -66,7 +66,6 @@ set timing on; WHENEVER SQLERROR CONTINUE; -drop table PMN_LABNORMAL; drop table DEMOGRAPHIC; drop table ENROLLMENT; drop table ENCOUNTER; From 3828e24da97a94ee309bdd66008ba4981e663cdb Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 5 May 2016 13:58:29 -0500 Subject: [PATCH 130/507] Updated SQL to reflect the correction of PCORI_BASECODE/PCORI_SPECIMEN_SOURCE switch in CTL creation --- Oracle/pcornet_mapping.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 13dca3a..986e9db 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -430,7 +430,7 @@ commit; insert into "&&i2b2_meta_schema".PCORNET_LAB with lab_map as ( - select distinct lab.c_hlevel, lab.c_path, lab.pcori_basecode, trim(CHR(13) from lab.pcori_specimen_source) as pcori_specimen_source + select distinct lab.c_hlevel, lab.c_path, lab.pcori_specimen_source, trim(CHR(13) from lab.pcori_basecode) as pcori_basecode from pcornet_lab lab inner JOIN pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname inner join pmn_labnormal norm on ont_parent.c_basecode=norm.LAB_NAME @@ -439,7 +439,7 @@ with lab_map as ( local_loinc_terms as ( select lm.C_HLEVEL, lt.C_FULLNAME, lm.C_PATH, lm.PCORI_BASECODE, lm.pcori_specimen_source from "&&i2b2_meta_schema"."&&terms_table" lt, lab_map lm - where lm.pcori_specimen_source=replace(lt.c_basecode, 'LOINC:', '') + where lm.pcori_basecode=replace(lt.c_basecode, 'LOINC:', '') and lt.c_fullname like '\i2b2\Laboratory Tests\%' and lt.c_basecode like 'LOINC:%' ) select From 6d91bbb0ef5211330aeb211fece7457f04cd61ac Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 9 May 2016 13:38:35 -0500 Subject: [PATCH 131/507] Forgot to switch the column order in this statement as well. --- Oracle/pcornet_mapping.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 986e9db..4227079 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -428,7 +428,7 @@ commit; /* Add relevent nodes from local i2b2 lab hierarchy to PCORNet Labs hierarchy. */ -insert into "&&i2b2_meta_schema".PCORNET_LAB +insert into PCORNET_LAB with lab_map as ( select distinct lab.c_hlevel, lab.c_path, lab.pcori_specimen_source, trim(CHR(13) from lab.pcori_basecode) as pcori_basecode from pcornet_lab lab @@ -468,8 +468,8 @@ select lt.M_EXCLUSION_CD, llt.C_PATH, lt.C_SYMBOL, - llt.PCORI_BASECODE, - llt.pcori_specimen_source + llt.pcori_specimen_source, + llt.PCORI_BASECODE from "&&i2b2_meta_schema"."&&terms_table" lt, local_loinc_terms llt where lt.c_fullname like llt.c_fullname||'%\' ; From 93b9272e493ca332de74d81726cce820da0e6612 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 12 May 2016 09:01:18 -0500 Subject: [PATCH 132/507] Temp fix for errors generated by inserting non-standard values into RESULT_QUAL. --- Oracle/PCORNetLoader_ora.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index ac66ea8..e1db772 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1275,7 +1275,8 @@ m.start_date SPECIMEN_DATE, to_char(m.start_date,'HH:MI') SPECIMEN_TIME, m.end_date RESULT_DATE, to_char(m.end_date,'HH:MI') RESULT_TIME, -CASE WHEN m.ValType_Cd='T' THEN NVL(nullif(m.TVal_Char,''),'NI') ELSE 'NI' END RESULT_QUAL, -- TODO: Should be a standardized value +--CASE WHEN m.ValType_Cd='T' THEN NVL(nullif(m.TVal_Char,''),'NI') ELSE 'NI' END RESULT_QUAL, -- TODO: Should be a standardized value +'NI' RESULT_QUAL, -- Temporary fix for KUMC CASE WHEN m.ValType_Cd='N' THEN m.NVAL_NUM ELSE null END RESULT_NUM, CASE WHEN m.ValType_Cd='N' THEN (CASE NVL(nullif(m.TVal_Char,''),'NI') WHEN 'E' THEN 'EQ' WHEN 'NE' THEN 'OT' WHEN 'L' THEN 'LT' WHEN 'LE' THEN 'LE' WHEN 'G' THEN 'GT' WHEN 'GE' THEN 'GE' ELSE 'NI' END) ELSE 'TX' END RESULT_MODIFIER, NVL(m.Units_CD,'NI') RESULT_UNIT, -- TODO: Should be standardized units From 129bf9bcefba354c1648ffae539eec623e14bd9a Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 12 May 2016 12:27:55 -0500 Subject: [PATCH 133/507] Removed duplicate reference to Creatinine lab. --- Oracle/pmn_labnormal.csv | 1 - 1 file changed, 1 deletion(-) diff --git a/Oracle/pmn_labnormal.csv b/Oracle/pmn_labnormal.csv index 6d6277a..fd92568 100644 --- a/Oracle/pmn_labnormal.csv +++ b/Oracle/pmn_labnormal.csv @@ -5,7 +5,6 @@ LAB_NAME:CK,50,GE,236,LE LAB_NAME:CK_MB,,NI,,NI LAB_NAME:CK_MBI,,NI,,NI LAB_NAME:CREATININE,0,GE,1.6,LE -LAB_NAME:CREATININE,0,GE,1.6,LE LAB_NAME:HGB,12,GE,17.5,LE LAB_NAME:INR,0.8,GE,1.3,LE LAB_NAME:TROP_I,0,GE,0.49,LE From 146792a4b8f97a6e578c54d0bc653e1d47344166 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 12 May 2016 12:46:15 -0500 Subject: [PATCH 134/507] Added simple LAB_RESULT_CM SQL test. --- Oracle/cdm_transform_tests.sql | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 9707fbe..ff8374f 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -528,6 +528,38 @@ order by 2 ; */ + +/* Verify that LAB_RESULT_CM contains data associated with a least half of the +labs defined in PMN_LABLOCAL. +*/ +insert into test_cases (query_name, description, obs, by_value1, by_value2, record_pct, pass) +with partial_lab_counts as ( + select lab_name, count(lab_name) lab_count + from lab_result_cm + group by lab_name +), complete_lab_counts as( + select replace(pmn.lab_name, 'LAB_NAME:', '') lab_name, + case when lc.lab_count is null then 0 else lc.lab_count end lab_count + from pmn_labnormal pmn + left outer join partial_lab_counts lc on concat('LAB_NAME:', lc.lab_name) = pmn.lab_name +), total_lab_types as ( + select count(*) num_labs from complete_lab_counts +), present_lab_types as ( + select count(*) num_labs from complete_lab_counts + where lab_count > 0 +) +select +'LAB_RESULT_CM Test' query_name, +'LAB_RESULT_CM contains data associated with at least half of the defined labs' description, +rownum obs, +plt.num_labs by_value1, +tlt.num_labs by_values2, +round(plt.num_labs/tlt.num_labs, 4) record_pct, +case when (plt.num_labs/tlt.num_labs) > 0.5 then 1 else 0 end pass +from present_lab_types plt +cross join total_lab_types tlt +; + select case when count(*) > 0 then 1/0 else 1 end all_test_cases_pass from ( select * from test_cases where pass = 0 ); From 0d00d3e100a1d1ea5fe4bf206b070875ec5f4589 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 17 May 2016 11:55:39 -0500 Subject: [PATCH 135/507] The pcornet_lab synonym isn't created before pcornet_mapping.sql is run. --- Oracle/pcornet_mapping.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 4227079..c54c635 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -428,11 +428,11 @@ commit; /* Add relevent nodes from local i2b2 lab hierarchy to PCORNet Labs hierarchy. */ -insert into PCORNET_LAB +insert into "&&i2b2_meta_schema".pcornet_lab with lab_map as ( select distinct lab.c_hlevel, lab.c_path, lab.pcori_specimen_source, trim(CHR(13) from lab.pcori_basecode) as pcori_basecode - from pcornet_lab lab - inner JOIN pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname + from "&&i2b2_meta_schema".pcornet_lab lab + inner JOIN "&&i2b2_meta_schema".pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname inner join pmn_labnormal norm on ont_parent.c_basecode=norm.LAB_NAME where lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' ), From f19e20beca5a2fcc7e4d3894634bac2dac85eb1d Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 23 May 2016 13:46:17 -0500 Subject: [PATCH 136/507] Setting RESULT_UNIT to NI until solution can be developed. --- Oracle/PCORNetLoader_ora.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index e1db772..08e144b 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1280,6 +1280,7 @@ to_char(m.end_date,'HH:MI') RESULT_TIME, CASE WHEN m.ValType_Cd='N' THEN m.NVAL_NUM ELSE null END RESULT_NUM, CASE WHEN m.ValType_Cd='N' THEN (CASE NVL(nullif(m.TVal_Char,''),'NI') WHEN 'E' THEN 'EQ' WHEN 'NE' THEN 'OT' WHEN 'L' THEN 'LT' WHEN 'LE' THEN 'LE' WHEN 'G' THEN 'GT' WHEN 'GE' THEN 'GE' ELSE 'NI' END) ELSE 'TX' END RESULT_MODIFIER, NVL(m.Units_CD,'NI') RESULT_UNIT, -- TODO: Should be standardized units +'NI' RESULT_UNIT, -- Temporary fix for KUMC nullif(norm.NORM_RANGE_LOW,'') NORM_RANGE_LOW ,norm.NORM_MODIFIER_LOW, nullif(norm.NORM_RANGE_HIGH,'') NORM_RANGE_HIGH From 0a1446dfaf16b7d76bca3e2241242ef90cd9b44a Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 23 May 2016 13:54:10 -0500 Subject: [PATCH 137/507] Forgot to comment out the orignal RESULT_UNIT line. --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 08e144b..e7738fd 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1279,7 +1279,7 @@ to_char(m.end_date,'HH:MI') RESULT_TIME, 'NI' RESULT_QUAL, -- Temporary fix for KUMC CASE WHEN m.ValType_Cd='N' THEN m.NVAL_NUM ELSE null END RESULT_NUM, CASE WHEN m.ValType_Cd='N' THEN (CASE NVL(nullif(m.TVal_Char,''),'NI') WHEN 'E' THEN 'EQ' WHEN 'NE' THEN 'OT' WHEN 'L' THEN 'LT' WHEN 'LE' THEN 'LE' WHEN 'G' THEN 'GT' WHEN 'GE' THEN 'GE' ELSE 'NI' END) ELSE 'TX' END RESULT_MODIFIER, -NVL(m.Units_CD,'NI') RESULT_UNIT, -- TODO: Should be standardized units +--NVL(m.Units_CD,'NI') RESULT_UNIT, -- TODO: Should be standardized units 'NI' RESULT_UNIT, -- Temporary fix for KUMC nullif(norm.NORM_RANGE_LOW,'') NORM_RANGE_LOW ,norm.NORM_MODIFIER_LOW, From 9cbd399e60a230b93b31f451fdbfd0e54a5b8904 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 8 Jun 2016 12:56:34 -0500 Subject: [PATCH 138/507] Added/updated fixes for RESULT_QUAL, RESULT_UNIT, and RAW_RESULT fields. --- Oracle/PCORNetLoader_ora.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index e7738fd..6ea04aa 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1276,11 +1276,11 @@ to_char(m.start_date,'HH:MI') SPECIMEN_TIME, m.end_date RESULT_DATE, to_char(m.end_date,'HH:MI') RESULT_TIME, --CASE WHEN m.ValType_Cd='T' THEN NVL(nullif(m.TVal_Char,''),'NI') ELSE 'NI' END RESULT_QUAL, -- TODO: Should be a standardized value -'NI' RESULT_QUAL, -- Temporary fix for KUMC +'NI' RESULT_QUAL, -- Local fix for KUMC (temp) CASE WHEN m.ValType_Cd='N' THEN m.NVAL_NUM ELSE null END RESULT_NUM, CASE WHEN m.ValType_Cd='N' THEN (CASE NVL(nullif(m.TVal_Char,''),'NI') WHEN 'E' THEN 'EQ' WHEN 'NE' THEN 'OT' WHEN 'L' THEN 'LT' WHEN 'LE' THEN 'LE' WHEN 'G' THEN 'GT' WHEN 'GE' THEN 'GE' ELSE 'NI' END) ELSE 'TX' END RESULT_MODIFIER, --NVL(m.Units_CD,'NI') RESULT_UNIT, -- TODO: Should be standardized units -'NI' RESULT_UNIT, -- Temporary fix for KUMC +CASE WHEN INSTR(m.Units_CD, '%') > 0 THEN 'PERCENT' WHEN m.Units_CD IS NULL THEN NVL(m.Units_CD,'NI') ELSE TRIM(REPLACE(UPPER(m.Units_CD), '(CALC)', '')) end RESULT_UNIT, -- Local fix for KUMC nullif(norm.NORM_RANGE_LOW,'') NORM_RANGE_LOW ,norm.NORM_MODIFIER_LOW, nullif(norm.NORM_RANGE_HIGH,'') NORM_RANGE_HIGH @@ -1289,7 +1289,8 @@ CASE NVL(nullif(m.VALUEFLAG_CD,''),'NI') WHEN 'H' THEN 'AH' WHEN 'L' THEN 'AL' W NULL RAW_LAB_NAME, NULL RAW_LAB_CODE, NULL RAW_PANEL, -CASE WHEN m.ValType_Cd='T' THEN m.TVal_Char ELSE to_char(m.NVal_Num) END RAW_RESULT, +--CASE WHEN m.ValType_Cd='T' THEN m.TVal_Char ELSE to_char(m.NVal_Num) END RAW_RESULT, +CASE WHEN m.ValType_Cd='T' THEN substr(m.TVal_Char, 1, 50) ELSE to_char(m.NVal_Num) END RAW_RESULT, -- Local fix for KUMC NULL RAW_UNIT, NULL RAW_ORDER_DEPT, NULL RAW_FACILITY_CODE From 211e1761744a38f9b58e0a2744c3963bc198e48c Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Thu, 16 Jun 2016 15:19:04 -0500 Subject: [PATCH 139/507] Extract the frequency code from the pcori_basecode column. - --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 6ea04aa..113d70e 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1471,7 +1471,7 @@ insert into prescribing ( -- ,RAW_RXNORM_CUI ) select distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH:MI'), m.start_date start_date, m.end_date, mo.pcori_cui - ,quantity.nval_num quantity, refills.nval_num refills, supply.nval_num supply, freq.pcori_basecode frequency, + ,quantity.nval_num quantity, refills.nval_num refills, supply.nval_num supply, substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) frequency, substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) basis from i2b2fact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode inner join encounter enc on enc.encounterid = m.encounter_Num From 66623a6be08e6c4df3af3827c92a57e4e78522a6 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Thu, 16 Jun 2016 15:34:01 -0500 Subject: [PATCH 140/507] Populate pcornet_med terms with local RX frequency modifier leaves - Remove erroneous comment. --- Oracle/pcornet_mapping.sql | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index c54c635..e34132f 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -333,6 +333,31 @@ set med.c_visualattributes = 'DAE' where c_fullname in ( join "&&i2b2_meta_schema".PCORNET_MED med on med.c_fullname = pm.pcori_path where med.c_fullname like '\PCORI_MOD\%' ); + +/* RX Frequency modifiers +*/ +create or replace view rx_frequency_strs as +select '\PCORI_MOD\RX_FREQUENCY\' freq_mod_path, 'RX_FREQUENCY:' freq_mod_pfx from dual; + +delete +from "&&i2b2_meta_schema".PCORNET_MED pm +where pm.c_fullname like (select strs.freq_mod_path || '%' from rx_frequency_strs strs); + +insert into pcornet_med +select + c_hlevel, c_fullname, c_name, c_synonym_cd, c_visualattributes, c_totalnum, + c_basecode, c_metadataxml, c_facttablecolumn, c_tablename, c_columnname, + c_columndatatype, c_operator, c_dimcode, c_comment, c_tooltip, m_applied_path, + update_date, download_date, import_date, sourcesystem_cd, valuetype_cd, + m_exclusion_cd, c_path, c_symbol, + strs.freq_mod_pfx || substr(ht.c_fullname, length(strs.freq_mod_path) + 1, 2) pcori_basecode, + null pcori_cui, null pcori_ndc +from blueheronmetadata.heron_terms ht +cross join rx_frequency_strs strs +where ht.c_fullname like (select strs.freq_mod_path || '%' from rx_frequency_strs) +and ht.m_exclusion_cd is null +and ht.c_basecode is not null; + insert into "&&i2b2_meta_schema".PCORNET_MED with @@ -382,7 +407,7 @@ select delete from "&&i2b2_meta_schema".PCORNET_MED where sourcesystem_cd='MAPPING'; --- Other diagnosis mappings such as modifiers for PDX, DX_SOURCE. + insert into "&&i2b2_meta_schema".PCORNET_MED with med_mod_mapping as ( select distinct pcori_path, i2b2.c_fullname, i2b2.c_basecode, i2b2.c_name From 412f3dc74df80f09fffa778ac4430a1c09f6e308 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Thu, 16 Jun 2016 15:39:37 -0500 Subject: [PATCH 141/507] Replace reference to local schema/table with substitution variables. --- Oracle/pcornet_mapping.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index e34132f..b1cd083 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -352,7 +352,7 @@ select m_exclusion_cd, c_path, c_symbol, strs.freq_mod_pfx || substr(ht.c_fullname, length(strs.freq_mod_path) + 1, 2) pcori_basecode, null pcori_cui, null pcori_ndc -from blueheronmetadata.heron_terms ht +from "&&i2b2_meta_schema"."&&terms_table" ht cross join rx_frequency_strs strs where ht.c_fullname like (select strs.freq_mod_path || '%' from rx_frequency_strs) and ht.m_exclusion_cd is null From 0a7dce08a6027e8625a32e40acce55e51ecbfc67 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Fri, 29 Jul 2016 15:41:48 -0500 Subject: [PATCH 142/507] Added PATID and ENCOUNTERID indexes to tables that contain these fields. --- Oracle/PCORNetLoader_ora.sql | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 6ea04aa..0cee630 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -873,6 +873,9 @@ FETCH getsql INTO sqltext; COMMIT; END LOOP; CLOSE getsql; + +execute immediate 'create index demographic_patid on pcornet_cdm.demographic (PATID)'; + end PCORNetDemographic; / @@ -924,6 +927,9 @@ left outer join inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num; +execute immediate 'create index encounter_patid on encounter (PATID)'; +execute immediate 'create index encounter_encounterid on encounter (ENCOUNTERID)'; + end PCORNetEncounter; / @@ -981,6 +987,8 @@ sqltext := 'insert into diagnosis (patid, encounterid, enc_type, admit_date, p PMN_EXECUATESQL(sqltext); +execute immediate 'create index diagnosis_patid on diagnosis (PATID)'; +execute immediate 'create index diagnosis_encounterid on diagnosis (ENCOUNTERID)'; end PCORNetDiagnosis; / @@ -1024,6 +1032,9 @@ sqltext := 'insert into condition (patid, encounterid, report_date, resolve_date PMN_EXECUATESQL(sqltext); +execute immediate 'create index condition_patid on condition (PATID)'; +execute immediate 'create index condition_encounterid on condition (ENCOUNTERID)'; + end PCORNetCondition; / @@ -1046,6 +1057,9 @@ from i2b2fact fact inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd where pr.c_fullname like '\PCORI\PROCEDURE\%'; +execute immediate 'create index procedures_patid on procedures (PATID)'; +execute immediate 'create index procedures_encounterid on procedures (ENCOUNTERID)'; + end PCORNetProcedure; / @@ -1130,6 +1144,9 @@ where ht is not null or tobacco is not null group by patid, encounterid, measure_date, measure_time, admit_date) y; +execute immediate 'create index vital_patid on vital (PATID)'; +execute immediate 'create index vital_encounterid on vital (ENCOUNTERID)'; + end PCORNetVital; / @@ -1161,6 +1178,7 @@ from enrolled enr join i2b2visit visit on enr.patient_num = visit.patient_num group by visit.patient_num; +execute immediate 'create index enrollment_patid on enrollment (PATID)'; end PCORNetEnroll; / @@ -1322,6 +1340,9 @@ WHERE m.ValType_Cd in ('N','T') and ont_parent.C_BASECODE LIKE 'LAB_NAME%' -- Exclude non-pcori labs and m.MODIFIER_CD='@'; +execute immediate 'create index lab_result_cm_patid on lab_result_cm (PATID)'; +execute immediate 'create index lab_result_cm_encounterid on lab_result_cm (ENCOUNTERID)'; + END PCORNetLabResultCM; / @@ -1499,6 +1520,8 @@ inner join encounter enc on enc.encounterid = m.encounter_Num where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); +execute immediate 'create index prescribing_patid on prescribing (PATID)'; +execute immediate 'create index prescribing_encounterid on prescribing (ENCOUNTERID)'; end PCORNetPrescribing; / @@ -1584,6 +1607,8 @@ inner join encounter enc on enc.encounterid = m.encounter_Num group by m.encounter_num ,m.patient_num, m.start_date, mo.pcori_ndc; +execute immediate 'create index dispensing_patid on dispensing (PATID)'; + end PCORNetDispensing; / From 79613ee75ff9e7923f54ffc4aebd13866f05a5d5 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 1 Aug 2016 09:51:56 -0500 Subject: [PATCH 143/507] Added DROP INDEX statements to prevent errors if indexes already exist. --- Oracle/PCORNetLoader_ora.sql | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 0cee630..353172b 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -874,7 +874,8 @@ FETCH getsql INTO sqltext; END LOOP; CLOSE getsql; -execute immediate 'create index demographic_patid on pcornet_cdm.demographic (PATID)'; +execute immediate 'drop index demographic_patid'; +execute immediate 'create index demographic_patid on demographic (PATID)'; end PCORNetDemographic; / @@ -927,7 +928,9 @@ left outer join inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num; +execute immediate 'drop index encounter_patid'; execute immediate 'create index encounter_patid on encounter (PATID)'; +execute immediate 'drop index encounter_encounterid'; execute immediate 'create index encounter_encounterid on encounter (ENCOUNTERID)'; end PCORNetEncounter; @@ -987,7 +990,9 @@ sqltext := 'insert into diagnosis (patid, encounterid, enc_type, admit_date, p PMN_EXECUATESQL(sqltext); +execute immediate 'drop index diagnosis_patid'; execute immediate 'create index diagnosis_patid on diagnosis (PATID)'; +execute immediate 'drop index diagnosis_encounterid'; execute immediate 'create index diagnosis_encounterid on diagnosis (ENCOUNTERID)'; end PCORNetDiagnosis; @@ -1032,7 +1037,9 @@ sqltext := 'insert into condition (patid, encounterid, report_date, resolve_date PMN_EXECUATESQL(sqltext); +execute immediate 'drop index condition_patid'; execute immediate 'create index condition_patid on condition (PATID)'; +execute immediate 'drop index condition_encounterid'; execute immediate 'create index condition_encounterid on condition (ENCOUNTERID)'; end PCORNetCondition; @@ -1057,7 +1064,9 @@ from i2b2fact fact inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd where pr.c_fullname like '\PCORI\PROCEDURE\%'; +execute immediate 'drop index procedures_patid'; execute immediate 'create index procedures_patid on procedures (PATID)'; +execute immediate 'drop index procedures_encounterid'; execute immediate 'create index procedures_encounterid on procedures (ENCOUNTERID)'; end PCORNetProcedure; @@ -1144,7 +1153,9 @@ where ht is not null or tobacco is not null group by patid, encounterid, measure_date, measure_time, admit_date) y; +execute immediate 'drop index vital_patid'; execute immediate 'create index vital_patid on vital (PATID)'; +execute immediate 'drop index vital_encounterid'; execute immediate 'create index vital_encounterid on vital (ENCOUNTERID)'; end PCORNetVital; @@ -1178,6 +1189,7 @@ from enrolled enr join i2b2visit visit on enr.patient_num = visit.patient_num group by visit.patient_num; +execute immediate 'drop index enrollment_patid'; execute immediate 'create index enrollment_patid on enrollment (PATID)'; end PCORNetEnroll; @@ -1340,7 +1352,9 @@ WHERE m.ValType_Cd in ('N','T') and ont_parent.C_BASECODE LIKE 'LAB_NAME%' -- Exclude non-pcori labs and m.MODIFIER_CD='@'; +execute immediate 'drop index lab_result_cm_patid'; execute immediate 'create index lab_result_cm_patid on lab_result_cm (PATID)'; +execute immediate 'drop index lab_result_cm_encounterid'; execute immediate 'create index lab_result_cm_encounterid on lab_result_cm (ENCOUNTERID)'; END PCORNetLabResultCM; @@ -1520,7 +1534,9 @@ inner join encounter enc on enc.encounterid = m.encounter_Num where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); +execute immediate 'drop index prescribing_patid'; execute immediate 'create index prescribing_patid on prescribing (PATID)'; +execute immediate 'drop index prescribing_encounterid'; execute immediate 'create index prescribing_encounterid on prescribing (ENCOUNTERID)'; end PCORNetPrescribing; @@ -1607,6 +1623,7 @@ inner join encounter enc on enc.encounterid = m.encounter_Num group by m.encounter_num ,m.patient_num, m.start_date, mo.pcori_ndc; +execute immediate 'drop index dispensing_patid'; execute immediate 'create index dispensing_patid on dispensing (PATID)'; end PCORNetDispensing; From dc6d35587d0cafe877f705a419464162ac954df3 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 1 Aug 2016 12:49:23 -0500 Subject: [PATCH 144/507] Use PMN_DROPSQL procedure to drop indexes to avoid throwing errors if the index doesn't exist. --- Oracle/PCORNetLoader_ora.sql | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 353172b..f2d583e 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -874,7 +874,7 @@ FETCH getsql INTO sqltext; END LOOP; CLOSE getsql; -execute immediate 'drop index demographic_patid'; +PMN_DROPSQL('drop index demographic_patid'); execute immediate 'create index demographic_patid on demographic (PATID)'; end PCORNetDemographic; @@ -928,9 +928,9 @@ left outer join inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num; -execute immediate 'drop index encounter_patid'; +PMN_DROPSQL('drop index encounter_patid'); execute immediate 'create index encounter_patid on encounter (PATID)'; -execute immediate 'drop index encounter_encounterid'; +PMN_DROPSQL('drop index encounter_encounterid'); execute immediate 'create index encounter_encounterid on encounter (ENCOUNTERID)'; end PCORNetEncounter; @@ -990,9 +990,9 @@ sqltext := 'insert into diagnosis (patid, encounterid, enc_type, admit_date, p PMN_EXECUATESQL(sqltext); -execute immediate 'drop index diagnosis_patid'; +PMN_DROPSQL('drop index diagnosis_patid'); execute immediate 'create index diagnosis_patid on diagnosis (PATID)'; -execute immediate 'drop index diagnosis_encounterid'; +PMN_DROPSQL('drop index diagnosis_encounterid'); execute immediate 'create index diagnosis_encounterid on diagnosis (ENCOUNTERID)'; end PCORNetDiagnosis; @@ -1037,9 +1037,9 @@ sqltext := 'insert into condition (patid, encounterid, report_date, resolve_date PMN_EXECUATESQL(sqltext); -execute immediate 'drop index condition_patid'; +PMN_DROPSQL('drop index condition_patid'); execute immediate 'create index condition_patid on condition (PATID)'; -execute immediate 'drop index condition_encounterid'; +PMN_DROPSQL('drop index condition_encounterid'); execute immediate 'create index condition_encounterid on condition (ENCOUNTERID)'; end PCORNetCondition; @@ -1064,9 +1064,9 @@ from i2b2fact fact inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd where pr.c_fullname like '\PCORI\PROCEDURE\%'; -execute immediate 'drop index procedures_patid'; +PMN_DROPSQL('drop index procedures_patid'); execute immediate 'create index procedures_patid on procedures (PATID)'; -execute immediate 'drop index procedures_encounterid'; +PMN_DROPSQL('drop index procedures_encounterid'); execute immediate 'create index procedures_encounterid on procedures (ENCOUNTERID)'; end PCORNetProcedure; @@ -1153,9 +1153,9 @@ where ht is not null or tobacco is not null group by patid, encounterid, measure_date, measure_time, admit_date) y; -execute immediate 'drop index vital_patid'; +PMN_DROPSQL('drop index vital_patid'); execute immediate 'create index vital_patid on vital (PATID)'; -execute immediate 'drop index vital_encounterid'; +PMN_DROPSQL('drop index vital_encounterid'); execute immediate 'create index vital_encounterid on vital (ENCOUNTERID)'; end PCORNetVital; @@ -1189,7 +1189,7 @@ from enrolled enr join i2b2visit visit on enr.patient_num = visit.patient_num group by visit.patient_num; -execute immediate 'drop index enrollment_patid'; +PMN_DROPSQL('drop index enrollment_patid'); execute immediate 'create index enrollment_patid on enrollment (PATID)'; end PCORNetEnroll; @@ -1352,9 +1352,9 @@ WHERE m.ValType_Cd in ('N','T') and ont_parent.C_BASECODE LIKE 'LAB_NAME%' -- Exclude non-pcori labs and m.MODIFIER_CD='@'; -execute immediate 'drop index lab_result_cm_patid'; +PMN_DROPSQL('drop index lab_result_cm_patid'); execute immediate 'create index lab_result_cm_patid on lab_result_cm (PATID)'; -execute immediate 'drop index lab_result_cm_encounterid'; +PMN_DROPSQL('drop index lab_result_cm_encounterid'); execute immediate 'create index lab_result_cm_encounterid on lab_result_cm (ENCOUNTERID)'; END PCORNetLabResultCM; @@ -1534,9 +1534,9 @@ inner join encounter enc on enc.encounterid = m.encounter_Num where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); -execute immediate 'drop index prescribing_patid'; +PMN_DROPSQL('drop index prescribing_patid'); execute immediate 'create index prescribing_patid on prescribing (PATID)'; -execute immediate 'drop index prescribing_encounterid'; +PMN_DROPSQL('drop index prescribing_encounterid'); execute immediate 'create index prescribing_encounterid on prescribing (ENCOUNTERID)'; end PCORNetPrescribing; @@ -1623,7 +1623,7 @@ inner join encounter enc on enc.encounterid = m.encounter_Num group by m.encounter_num ,m.patient_num, m.start_date, mo.pcori_ndc; -execute immediate 'drop index dispensing_patid'; +PMN_DROPSQL('drop index dispensing_patid'); execute immediate 'create index dispensing_patid on dispensing (PATID)'; end PCORNetDispensing; From 3d0765f283bd8b5dec771d5b6f57d5da6bb49f06 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 1 Aug 2016 14:27:57 -0500 Subject: [PATCH 145/507] Copy medication facts into their own table - reference while building prescribing. - Putting medication facts in their own table will hopefully reduce needed system resources when building the prescribing table. --- Oracle/PCORNetLoader_ora.sql | 14 ++++--- Oracle/observation_fact_meds.sql | 65 ++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 Oracle/observation_fact_meds.sql diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index a6efff4..2e83aa1 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -66,6 +66,8 @@ END PMN_ExecuateSQL; CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT / +CREATE OR REPLACE SYNONYM I2B2MEDFACT FOR OBSERVATION_FACT_MEDS +/ BEGIN PMN_DROPSQL('DROP TABLE i2b2patient_list'); @@ -1441,7 +1443,7 @@ begin PMN_DROPSQL('DROP TABLE basis'); sqltext := 'create table basis as '|| -'(select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2fact basis '|| +'(select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2medfact basis '|| ' inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num '|| ' join pcornet_med basiscode '|| ' on basis.modifier_cd = basiscode.c_basecode '|| @@ -1450,7 +1452,7 @@ PMN_EXECUATESQL(sqltext); PMN_DROPSQL('DROP TABLE freq'); sqltext := 'create table freq as '|| -'(select pcori_basecode,encounter_num,concept_cd from i2b2fact freq '|| +'(select pcori_basecode,encounter_num,concept_cd from i2b2medfact freq '|| ' inner join encounter enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num '|| ' join pcornet_med freqcode '|| ' on freq.modifier_cd = freqcode.c_basecode '|| @@ -1459,7 +1461,7 @@ PMN_EXECUATESQL(sqltext); PMN_DROPSQL('DROP TABLE quantity'); sqltext := 'create table quantity as '|| -'(select nval_num,encounter_num,concept_cd from i2b2fact quantity '|| +'(select nval_num,encounter_num,concept_cd from i2b2medfact quantity '|| ' inner join encounter enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num '|| ' join pcornet_med quantitycode '|| ' on quantity.modifier_cd = quantitycode.c_basecode '|| @@ -1469,7 +1471,7 @@ PMN_EXECUATESQL(sqltext); PMN_DROPSQL('DROP TABLE refills'); sqltext := 'create table refills as '|| -'(select nval_num,encounter_num,concept_cd from i2b2fact refills '|| +'(select nval_num,encounter_num,concept_cd from i2b2medfact refills '|| ' inner join encounter enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num '|| ' join pcornet_med refillscode '|| ' on refills.modifier_cd = refillscode.c_basecode '|| @@ -1478,7 +1480,7 @@ PMN_EXECUATESQL(sqltext); PMN_DROPSQL('DROP TABLE supply'); sqltext := 'create table supply as '|| -'(select nval_num,encounter_num,concept_cd from i2b2fact supply '|| +'(select nval_num,encounter_num,concept_cd from i2b2medfact supply '|| ' inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| ' join pcornet_med supplycode '|| ' on supply.modifier_cd = supplycode.c_basecode '|| @@ -1508,7 +1510,7 @@ insert into prescribing ( select distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH:MI'), m.start_date start_date, m.end_date, mo.pcori_cui ,quantity.nval_num quantity, refills.nval_num refills, supply.nval_num supply, substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) frequency, substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) basis - from i2b2fact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode + from i2b2medfact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode inner join encounter enc on enc.encounterid = m.encounter_Num -- TODO: This join adds several minutes to the load - must be debugged diff --git a/Oracle/observation_fact_meds.sql b/Oracle/observation_fact_meds.sql new file mode 100644 index 0000000..0fe9f95 --- /dev/null +++ b/Oracle/observation_fact_meds.sql @@ -0,0 +1,65 @@ +/* Building the prescribing table uses a lot of temp tablespace (at KUMC at +least). The function that builds the prescribing table failed even when we +provided 1TB of space. The error we get is like: + +ERROR at line 1: +ORA-12801: error signaled in parallel query server P003 +ORA-01652: unable to extend temp segment by 32 in tablespace TEMP +ORA-06512: at "PCORNET_CDM.PCORNETPRESCRIBING", line 53 +ORA-06512: at "PCORNET_CDM.PCORNETLOADER", line 12 +ORA-06512: at line 2 + +So, let's make a separate fact table with just medication facts and have the +i2p-transform code reference that table for prescribing. + +We (KUMC) have over 2 billion rows in our fact table - only about 234 million are +medications. So, hopefully having a table that's an order of magnitude smaller +will help Oracle complete the query using less system resources. +*/ +whenever sqlerror continue; +drop index MED_OBS_FACT_ENC_NUM_BI; +drop index MED_OBS_FACT_PAT_NUM_BI; +drop index MED_OBS_FACT_CON_CODE_BI; +drop index MED_OBS_FACT_VALTYP_CD_BI; +drop index MED_OBS_FACT_NVAL_NUM_BI; +drop index MED_OBS_FACT_MOD_CODE_BI; +whenever sqlerror exit; + +-- Drop/create a fact table for medications +whenever sqlerror continue; +drop table observation_fact_meds; +whenever sqlerror exit; + +create table observation_fact_meds as + select * from "&&i2b2_data_schema".observation_fact where 1=0; + +-- Insert medication facts +insert into observation_fact_meds +select * from "&&i2b2_data_schema".observation_fact +where concept_cd like 'KUH|MEDICATION_ID:%'; +commit; + +-- Build indexes like https://informatics.kumc.edu/work/browser/heron_load/i2b2_facts_index.sql +create bitmap index MED_OBS_FACT_ENC_NUM_BI + on observation_fact_meds (ENCOUNTER_NUM) + nologging parallel 6; + +create bitmap index MED_OBS_FACT_PAT_NUM_BI + on observation_fact_meds (PATIENT_NUM) + nologging parallel 6; + +create bitmap index MED_OBS_FACT_CON_CODE_BI + on observation_fact_meds (CONCEPT_CD) + nologging parallel 6; + +create bitmap index MED_OBS_FACT_VALTYP_CD_BI + on observation_fact_meds (VALTYPE_CD) + nologging parallel 6; + +create bitmap index MED_OBS_FACT_NVAL_NUM_BI + on observation_fact_meds (NVAL_NUM) + nologging parallel 6; + +create bitmap index MED_OBS_FACT_MOD_CODE_BI + on observation_fact_meds (MODIFIER_CD) + nologging parallel 6; From 084498bd230793eb2bfc9bceb07437cb8e32ea16 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 1 Aug 2016 14:37:54 -0500 Subject: [PATCH 146/507] Add early test to make sure the medication fact table is populated. --- Oracle/run-i2p-transform.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 77c08e7..e6ff831 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -54,6 +54,11 @@ select case when qty = 0 then 1/0 else 1 end inout_cd_populated from ( -- Make sure the RXNorm mapping table exists select rxcui from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id where 1=0; +-- Make sure the observation fact medication table is populated +select case when qty > 0 then 1 else 1/0 end obs_fact_meds_populated from ( + select count(*) qty from observation_fact_meds + ); + EOF From 8f80b4d5d37bb4a9120906fe2a159cc95aa34460 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 2 Aug 2016 08:51:58 -0500 Subject: [PATCH 147/507] Drop indexes at the beginning of each procedure rather than the end. - Inserting with existing indexes leads to poor performance in practice. - Thanks to Mike for review. --- Oracle/PCORNetLoader_ora.sql | 47 +++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 2e83aa1..4bf579c 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -866,6 +866,8 @@ union --6 -- NS, NR, nH begin pcornet_popcodelist; +PMN_DROPSQL('drop index demographic_patid'); + OPEN getsql; LOOP FETCH getsql INTO sqltext; @@ -876,7 +878,6 @@ FETCH getsql INTO sqltext; END LOOP; CLOSE getsql; -PMN_DROPSQL('drop index demographic_patid'); execute immediate 'create index demographic_patid on demographic (PATID)'; end PCORNetDemographic; @@ -896,6 +897,9 @@ create or replace procedure PCORNetEncounter as sqltext varchar2(4000); begin +PMN_DROPSQL('drop index encounter_patid'); +PMN_DROPSQL('drop index encounter_encounterid'); + insert into encounter(PATID,ENCOUNTERID,admit_date ,ADMIT_TIME , DISCHARGE_DATE ,DISCHARGE_TIME ,PROVIDERID ,FACILITY_LOCATION ,ENC_TYPE ,FACILITYID ,DISCHARGE_DISPOSITION , @@ -930,9 +934,7 @@ left outer join inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num; -PMN_DROPSQL('drop index encounter_patid'); execute immediate 'create index encounter_patid on encounter (PATID)'; -PMN_DROPSQL('drop index encounter_encounterid'); execute immediate 'create index encounter_encounterid on encounter (ENCOUNTERID)'; end PCORNetEncounter; @@ -945,6 +947,9 @@ create or replace procedure PCORNetDiagnosis as sqltext varchar2(4000); begin +PMN_DROPSQL('drop index diagnosis_patid'); +PMN_DROPSQL('drop index diagnosis_encounterid'); + PMN_DROPSQL('DROP TABLE sourcefact'); sqltext := 'create table sourcefact as '|| @@ -992,9 +997,7 @@ sqltext := 'insert into diagnosis (patid, encounterid, enc_type, admit_date, p PMN_EXECUATESQL(sqltext); -PMN_DROPSQL('drop index diagnosis_patid'); execute immediate 'create index diagnosis_patid on diagnosis (PATID)'; -PMN_DROPSQL('drop index diagnosis_encounterid'); execute immediate 'create index diagnosis_encounterid on diagnosis (ENCOUNTERID)'; end PCORNetDiagnosis; @@ -1008,6 +1011,8 @@ create or replace procedure PCORNetCondition as sqltext varchar2(4000); begin +PMN_DROPSQL('drop index condition_patid'); +PMN_DROPSQL('drop index condition_encounterid'); PMN_DROPSQL('DROP TABLE sourcefact2'); @@ -1039,9 +1044,7 @@ sqltext := 'insert into condition (patid, encounterid, report_date, resolve_date PMN_EXECUATESQL(sqltext); -PMN_DROPSQL('drop index condition_patid'); execute immediate 'create index condition_patid on condition (PATID)'; -PMN_DROPSQL('drop index condition_encounterid'); execute immediate 'create index condition_encounterid on condition (ENCOUNTERID)'; end PCORNetCondition; @@ -1055,6 +1058,10 @@ end PCORNetCondition; create or replace procedure PCORNetProcedure as begin + +PMN_DROPSQL('drop index procedures_patid'); +PMN_DROPSQL('drop index procedures_encounterid'); + insert into procedures( patid, encounterid, enc_type, admit_date, px_date, providerid, px, px_type, px_source) select distinct fact.patient_num, enc.encounterid, enc.enc_type, enc.admit_date, fact.start_date, @@ -1066,9 +1073,7 @@ from i2b2fact fact inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd where pr.c_fullname like '\PCORI\PROCEDURE\%'; -PMN_DROPSQL('drop index procedures_patid'); execute immediate 'create index procedures_patid on procedures (PATID)'; -PMN_DROPSQL('drop index procedures_encounterid'); execute immediate 'create index procedures_encounterid on procedures (ENCOUNTERID)'; end PCORNetProcedure; @@ -1082,6 +1087,10 @@ end PCORNetProcedure; create or replace procedure PCORNetVital as begin + +PMN_DROPSQL('drop index vital_patid'); +PMN_DROPSQL('drop index vital_encounterid'); + -- jgk: I took out admit_date - it doesn't appear in the scheme. Now in SQLServer format - date, substring, name on inner select, no nested with. Added modifiers and now use only pathnames, not codes. insert into vital(patid, encounterid, measure_date, measure_time,vital_source,ht, wt, diastolic, systolic, original_bmi, bp_position,smoking,tobacco,tobacco_type) select patid, encounterid, to_date(measure_date,'rrrr-mm-dd') measure_date, measure_time,vital_source,ht, wt, diastolic, systolic, original_bmi, bp_position,smoking,tobacco, @@ -1155,9 +1164,7 @@ where ht is not null or tobacco is not null group by patid, encounterid, measure_date, measure_time, admit_date) y; -PMN_DROPSQL('drop index vital_patid'); execute immediate 'create index vital_patid on vital (PATID)'; -PMN_DROPSQL('drop index vital_encounterid'); execute immediate 'create index vital_encounterid on vital (ENCOUNTERID)'; end PCORNetVital; @@ -1171,6 +1178,8 @@ end PCORNetVital; create or replace procedure PCORNetEnroll as begin +PMN_DROPSQL('drop index enrollment_patid'); + INSERT INTO enrollment(PATID, ENR_START_DATE, ENR_END_DATE, CHART, ENR_BASIS) with pats_delta as ( -- If only one visit, visit_delta_days will be 0 @@ -1191,7 +1200,6 @@ from enrolled enr join i2b2visit visit on enr.patient_num = visit.patient_num group by visit.patient_num; -PMN_DROPSQL('drop index enrollment_patid'); execute immediate 'create index enrollment_patid on enrollment (PATID)'; end PCORNetEnroll; @@ -1238,6 +1246,10 @@ whenever sqlerror exit; create or replace procedure PCORNetLabResultCM as sqltext varchar2(4000); begin + +PMN_DROPSQL('drop index lab_result_cm_patid'); +PMN_DROPSQL('drop index lab_result_cm_encounterid'); + PMN_DROPSQL('DROP TABLE priority'); sqltext := 'create table priority as '|| @@ -1354,9 +1366,7 @@ WHERE m.ValType_Cd in ('N','T') and ont_parent.C_BASECODE LIKE 'LAB_NAME%' -- Exclude non-pcori labs and m.MODIFIER_CD='@'; -PMN_DROPSQL('drop index lab_result_cm_patid'); execute immediate 'create index lab_result_cm_patid on lab_result_cm (PATID)'; -PMN_DROPSQL('drop index lab_result_cm_encounterid'); execute immediate 'create index lab_result_cm_encounterid on lab_result_cm (ENCOUNTERID)'; END PCORNetLabResultCM; @@ -1441,6 +1451,9 @@ create or replace procedure PCORNetPrescribing as sqltext varchar2(4000); begin +PMN_DROPSQL('drop index prescribing_patid'); +PMN_DROPSQL('drop index prescribing_encounterid'); + PMN_DROPSQL('DROP TABLE basis'); sqltext := 'create table basis as '|| '(select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2medfact basis '|| @@ -1536,9 +1549,7 @@ inner join encounter enc on enc.encounterid = m.encounter_Num where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); -PMN_DROPSQL('drop index prescribing_patid'); execute immediate 'create index prescribing_patid on prescribing (PATID)'; -PMN_DROPSQL('drop index prescribing_encounterid'); execute immediate 'create index prescribing_encounterid on prescribing (ENCOUNTERID)'; end PCORNetPrescribing; @@ -1570,6 +1581,9 @@ whenever sqlerror exit; create or replace procedure PCORNetDispensing as sqltext varchar2(4000); begin + +PMN_DROPSQL('drop index dispensing_patid'); + PMN_DROPSQL('DROP TABLE supply'); sqltext := 'create table supply as '|| '(select nval_num,encounter_num,concept_cd from i2b2fact supply '|| @@ -1625,7 +1639,6 @@ inner join encounter enc on enc.encounterid = m.encounter_Num group by m.encounter_num ,m.patient_num, m.start_date, mo.pcori_ndc; -PMN_DROPSQL('drop index dispensing_patid'); execute immediate 'create index dispensing_patid on dispensing (PATID)'; end PCORNetDispensing; From 980bac0713149b6bc6c35633e0701a879413b906 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 3 Aug 2016 18:22:12 -0500 Subject: [PATCH 148/507] Add more indexes and gather table statistics to hopefully improve performance. - Added missing schema for insert into pcornet_med. --- Oracle/PCORNetLoader_ora.sql | 139 ++++++++++++++++++++++++++++++++++- Oracle/pcornet_mapping.sql | 2 +- 2 files changed, 139 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 4bf579c..85750c0 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -880,6 +880,14 @@ CLOSE getsql; execute immediate 'create index demographic_patid on demographic (PATID)'; +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. + tabname => 'DEMOGRAPHIC', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); + end PCORNetDemographic; / @@ -937,6 +945,14 @@ left outer join execute immediate 'create index encounter_patid on encounter (PATID)'; execute immediate 'create index encounter_encounterid on encounter (ENCOUNTERID)'; +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. + tabname => 'ENCOUNTER', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); + end PCORNetEncounter; / @@ -950,7 +966,7 @@ begin PMN_DROPSQL('drop index diagnosis_patid'); PMN_DROPSQL('drop index diagnosis_encounterid'); -PMN_DROPSQL('DROP TABLE sourcefact'); +PMN_DROPSQL('DROP TABLE sourcefact'); -- associated indexes will be dropped as well sqltext := 'create table sourcefact as '|| 'select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname '|| @@ -960,6 +976,15 @@ sqltext := 'create table sourcefact as '|| 'where dxsource.c_fullname like ''\PCORI_MOD\CONDITION_OR_DX\%'''; PMN_EXECUATESQL(sqltext); +execute immediate 'create index sourcefact_idx on sourcefact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; + +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'SOURCEFACT', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); PMN_DROPSQL('DROP TABLE pdxfact'); @@ -971,6 +996,15 @@ sqltext := 'create table pdxfact as '|| 'and dxsource.c_fullname like ''\PCORI_MOD\PDX\%'''; PMN_EXECUATESQL(sqltext); +execute immediate 'create index pdxfact_idx on pdxfact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; + +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'PDXFACT', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); sqltext := 'insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, pdx) '|| 'select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, factline.start_date, factline.provider_id, diag.pcori_basecode, '|| @@ -1000,6 +1034,14 @@ PMN_EXECUATESQL(sqltext); execute immediate 'create index diagnosis_patid on diagnosis (PATID)'; execute immediate 'create index diagnosis_encounterid on diagnosis (ENCOUNTERID)'; +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'DIAGNOSIS', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); + end PCORNetDiagnosis; / @@ -1076,6 +1118,14 @@ where pr.c_fullname like '\PCORI\PROCEDURE\%'; execute immediate 'create index procedures_patid on procedures (PATID)'; execute immediate 'create index procedures_encounterid on procedures (ENCOUNTERID)'; +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'PROCEDURES', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); + end PCORNetProcedure; / @@ -1167,6 +1217,14 @@ group by patid, encounterid, measure_date, measure_time, admit_date) y; execute immediate 'create index vital_patid on vital (PATID)'; execute immediate 'create index vital_encounterid on vital (ENCOUNTERID)'; +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'VITAL', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); + end PCORNetVital; / @@ -1202,6 +1260,14 @@ group by visit.patient_num; execute immediate 'create index enrollment_patid on enrollment (PATID)'; +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'ENROLLMENT', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); + end PCORNetEnroll; / @@ -1261,6 +1327,15 @@ sqltext := 'create table priority as '|| PMN_EXECUATESQL(sqltext); +execute immediate 'create index priority_idx on priority (patient_num, encounter_num, provider_id, concept_cd, start_date)'; + +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'PRIORITY', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); PMN_DROPSQL('DROP TABLE location'); sqltext := 'create table location as '|| @@ -1272,6 +1347,16 @@ sqltext := 'create table location as '|| PMN_EXECUATESQL(sqltext); +execute immediate 'create index location_idx on location (patient_num, encounter_num, provider_id, concept_cd, start_date)'; + +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'LOCATION', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); + INSERT INTO lab_result_cm (PATID @@ -1463,6 +1548,15 @@ sqltext := 'create table basis as '|| ' and basiscode.c_fullname like ''\PCORI_MOD\RX_BASIS\%'') '; PMN_EXECUATESQL(sqltext); +execute immediate 'create index basis_idx on basis (encounter_num, concept_cd)'; +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'BASIS', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); + PMN_DROPSQL('DROP TABLE freq'); sqltext := 'create table freq as '|| '(select pcori_basecode,encounter_num,concept_cd from i2b2medfact freq '|| @@ -1472,6 +1566,15 @@ sqltext := 'create table freq as '|| ' and freqcode.c_fullname like ''\PCORI_MOD\RX_FREQUENCY\%'') '; PMN_EXECUATESQL(sqltext); +execute immediate 'create index freq_idx on freq (encounter_num, concept_cd)'; +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'FREQ', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); + PMN_DROPSQL('DROP TABLE quantity'); sqltext := 'create table quantity as '|| '(select nval_num,encounter_num,concept_cd from i2b2medfact quantity '|| @@ -1481,6 +1584,15 @@ sqltext := 'create table quantity as '|| ' and quantitycode.c_fullname like ''\PCORI_MOD\RX_QUANTITY\'') '; PMN_EXECUATESQL(sqltext); + +execute immediate 'create index quantity_idx on quantity (encounter_num, concept_cd)'; +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'QUANTITY', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); PMN_DROPSQL('DROP TABLE refills'); sqltext := 'create table refills as '|| @@ -1491,6 +1603,15 @@ sqltext := 'create table refills as '|| ' and refillscode.c_fullname like ''\PCORI_MOD\RX_REFILLS\'') '; PMN_EXECUATESQL(sqltext); +execute immediate 'create index refills_idx on refills (encounter_num, concept_cd)'; +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'REFILLS', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); + PMN_DROPSQL('DROP TABLE supply'); sqltext := 'create table supply as '|| '(select nval_num,encounter_num,concept_cd from i2b2medfact supply '|| @@ -1500,6 +1621,14 @@ sqltext := 'create table supply as '|| ' and supplycode.c_fullname like ''\PCORI_MOD\RX_DAYS_SUPPLY\'') '; PMN_EXECUATESQL(sqltext); +execute immediate 'create index supply_idx on supply (encounter_num, concept_cd)'; +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'SUPPLY', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); -- insert data with outer joins to ensure all records are included even if some data elements are missing insert into prescribing ( @@ -1552,6 +1681,14 @@ where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR execute immediate 'create index prescribing_patid on prescribing (PATID)'; execute immediate 'create index prescribing_encounterid on prescribing (ENCOUNTERID)'; +DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', + tabname => 'PRESCRIBING', + estimate_percent => 50, + cascade => TRUE, + degree => 16 + ); + end PCORNetPrescribing; / diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index b1cd083..df73fb5 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -343,7 +343,7 @@ delete from "&&i2b2_meta_schema".PCORNET_MED pm where pm.c_fullname like (select strs.freq_mod_path || '%' from rx_frequency_strs strs); -insert into pcornet_med +insert into "&&i2b2_meta_schema".pcornet_med select c_hlevel, c_fullname, c_name, c_synonym_cd, c_visualattributes, c_totalnum, c_basecode, c_metadataxml, c_facttablecolumn, c_tablename, c_columnname, From 4ee9a3f9fa8717349eb268b1db522263a669de08 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Thu, 4 Aug 2016 11:06:23 -0500 Subject: [PATCH 149/507] Procedure to gather table statistics to consolidate parameters. --- Oracle/PCORNetLoader_ora.sql | 151 +++++++---------------------------- 1 file changed, 27 insertions(+), 124 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 85750c0..c07e523 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -20,6 +20,17 @@ --undef network_id; --undef network_name; +create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS + BEGIN + DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. + tabname => table_name, + estimate_percent => 50, -- Percentage picked somewhat arbitrarily + cascade => TRUE, + degree => 16 + ); +END GATHER_TABLE_STATS; +/ create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS BEGIN @@ -879,14 +890,7 @@ END LOOP; CLOSE getsql; execute immediate 'create index demographic_patid on demographic (PATID)'; - -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. - tabname => 'DEMOGRAPHIC', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('DEMOGRAPHIC'); end PCORNetDemographic; / @@ -944,14 +948,7 @@ left outer join execute immediate 'create index encounter_patid on encounter (PATID)'; execute immediate 'create index encounter_encounterid on encounter (ENCOUNTERID)'; - -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. - tabname => 'ENCOUNTER', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('ENCOUNTER'); end PCORNetEncounter; / @@ -977,14 +974,7 @@ sqltext := 'create table sourcefact as '|| PMN_EXECUATESQL(sqltext); execute immediate 'create index sourcefact_idx on sourcefact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; - -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'SOURCEFACT', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('SOURCEFACT'); PMN_DROPSQL('DROP TABLE pdxfact'); @@ -997,14 +987,7 @@ sqltext := 'create table pdxfact as '|| PMN_EXECUATESQL(sqltext); execute immediate 'create index pdxfact_idx on pdxfact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; - -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'PDXFACT', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('PDXFACT'); sqltext := 'insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, pdx) '|| 'select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, factline.start_date, factline.provider_id, diag.pcori_basecode, '|| @@ -1033,14 +1016,7 @@ PMN_EXECUATESQL(sqltext); execute immediate 'create index diagnosis_patid on diagnosis (PATID)'; execute immediate 'create index diagnosis_encounterid on diagnosis (ENCOUNTERID)'; - -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'DIAGNOSIS', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('DIAGNOSIS'); end PCORNetDiagnosis; / @@ -1117,14 +1093,7 @@ where pr.c_fullname like '\PCORI\PROCEDURE\%'; execute immediate 'create index procedures_patid on procedures (PATID)'; execute immediate 'create index procedures_encounterid on procedures (ENCOUNTERID)'; - -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'PROCEDURES', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('PROCEDURES'); end PCORNetProcedure; / @@ -1216,14 +1185,7 @@ group by patid, encounterid, measure_date, measure_time, admit_date) y; execute immediate 'create index vital_patid on vital (PATID)'; execute immediate 'create index vital_encounterid on vital (ENCOUNTERID)'; - -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'VITAL', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('VITAL'); end PCORNetVital; / @@ -1259,14 +1221,7 @@ join i2b2visit visit on enr.patient_num = visit.patient_num group by visit.patient_num; execute immediate 'create index enrollment_patid on enrollment (PATID)'; - -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'ENROLLMENT', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('ENROLLMENT'); end PCORNetEnroll; / @@ -1328,14 +1283,7 @@ sqltext := 'create table priority as '|| PMN_EXECUATESQL(sqltext); execute immediate 'create index priority_idx on priority (patient_num, encounter_num, provider_id, concept_cd, start_date)'; - -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'PRIORITY', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('PRIORITY'); PMN_DROPSQL('DROP TABLE location'); sqltext := 'create table location as '|| @@ -1348,15 +1296,7 @@ sqltext := 'create table location as '|| PMN_EXECUATESQL(sqltext); execute immediate 'create index location_idx on location (patient_num, encounter_num, provider_id, concept_cd, start_date)'; - -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'LOCATION', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); - +GATHER_TABLE_STATS('LOCATION'); INSERT INTO lab_result_cm (PATID @@ -1549,13 +1489,7 @@ sqltext := 'create table basis as '|| PMN_EXECUATESQL(sqltext); execute immediate 'create index basis_idx on basis (encounter_num, concept_cd)'; -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'BASIS', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('BASIS'); PMN_DROPSQL('DROP TABLE freq'); sqltext := 'create table freq as '|| @@ -1567,13 +1501,7 @@ sqltext := 'create table freq as '|| PMN_EXECUATESQL(sqltext); execute immediate 'create index freq_idx on freq (encounter_num, concept_cd)'; -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'FREQ', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('FREQ'); PMN_DROPSQL('DROP TABLE quantity'); sqltext := 'create table quantity as '|| @@ -1586,13 +1514,7 @@ sqltext := 'create table quantity as '|| PMN_EXECUATESQL(sqltext); execute immediate 'create index quantity_idx on quantity (encounter_num, concept_cd)'; -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'QUANTITY', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('QUANTITY'); PMN_DROPSQL('DROP TABLE refills'); sqltext := 'create table refills as '|| @@ -1604,13 +1526,7 @@ sqltext := 'create table refills as '|| PMN_EXECUATESQL(sqltext); execute immediate 'create index refills_idx on refills (encounter_num, concept_cd)'; -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'REFILLS', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('REFILLS'); PMN_DROPSQL('DROP TABLE supply'); sqltext := 'create table supply as '|| @@ -1622,13 +1538,7 @@ sqltext := 'create table supply as '|| PMN_EXECUATESQL(sqltext); execute immediate 'create index supply_idx on supply (encounter_num, concept_cd)'; -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'SUPPLY', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('SUPPLY'); -- insert data with outer joins to ensure all records are included even if some data elements are missing insert into prescribing ( @@ -1680,14 +1590,7 @@ where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR execute immediate 'create index prescribing_patid on prescribing (PATID)'; execute immediate 'create index prescribing_encounterid on prescribing (ENCOUNTERID)'; - -DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', - tabname => 'PRESCRIBING', - estimate_percent => 50, - cascade => TRUE, - degree => 16 - ); +GATHER_TABLE_STATS('PRESCRIBING'); end PCORNetPrescribing; / From cdfd9d81f463932eecd49072e749e1f4ae73959c Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Fri, 5 Aug 2016 09:40:12 -0500 Subject: [PATCH 150/507] Break apart admit source query to improve performance. - Add indexes/gather table statistics to help Oracle pick better plans. --- Oracle/heron_encounter_style.sql | 103 +++++++++++++++++++++---------- 1 file changed, 72 insertions(+), 31 deletions(-) diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 034f79e..fffe407 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -35,6 +35,21 @@ alter session set NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI'; TODO: push this back into HERON ETL */ + +--TODO: Consider trying to share this function with PCORNetLoad_ora.sql +create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS + BEGIN + DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. + tabname => table_name, + estimate_percent => 50, -- Percentage picked somewhat arbitrarily + cascade => TRUE, + degree => 16 + ); +END GATHER_TABLE_STATS; +/ + + merge into "&&i2b2_data_schema".visit_dimension vd using ( -- can we get it from UHC length_of_stay? @@ -113,6 +128,7 @@ drop table enc_type; whenever sqlerror exit; /* explore encounter mapping: where do our encounters come from? */ +/* select count(*), count(distinct encounter_num), encounter_ide_source from "&&i2b2_data_schema".encounter_mapping group by encounter_ide_source @@ -124,7 +140,7 @@ from "&&i2b2_data_schema".encounter_mapping group by encounter_ide_source order by 1 ; - +*/ merge into "&&i2b2_data_schema".visit_dimension vd using ( @@ -205,6 +221,10 @@ join ranks on pr.encounter_num = ranks.encounter_num and pr.pc_rank = ranks.pc_r ; create index discharge_disp_encnum_idx on discharge_disp(encounter_num); +begin +GATHER_TABLE_STATS('discharge_disp'); +end; +/ update "&&i2b2_data_schema".visit_dimension vd set discharge_disposition = coalesce(( @@ -238,6 +258,12 @@ select obs.encounter_num, et.pcori_code from "&&i2b2_data_schema".observation_fact obs join enc_status_codes et on et.concept_cd = obs.concept_cd; +create index enc_status_facts_encnum_idx on enc_status_facts(encounter_num); +begin +GATHER_TABLE_STATS('enc_status_facts'); +end; +/ + create table discharge_status as with ranks as ( @@ -262,6 +288,10 @@ join ranks on pr.encounter_num = ranks.encounter_num and pr.pc_rank = ranks.pc_r ; create index discharge_status_encnum_idx on discharge_status(encounter_num); +begin +GATHER_TABLE_STATS('discharge_status'); +end; +/ update "&&i2b2_data_schema".visit_dimension vd set discharge_status = coalesce(( @@ -277,41 +307,52 @@ whenever sqlerror continue; alter table "&&i2b2_data_schema".visit_dimension add ( admitting_source VARCHAR2(4 BYTE) ); +drop table admit_source_enc_code_rank; +drop table admit_source_priority_rank; whenever sqlerror exit; +create table admit_source_enc_code_rank as +with codes as ( + select + pm.pcori_path, cd.concept_cd, cd.name_char, + replace(pe.pcori_basecode, 'ADMITTING_SOURCE:', '') pcori_code + from + pcornet_mapping pm + join blueherondata.concept_dimension cd on cd.concept_path like pm.local_path || '%' + join blueheronmetadata.pcornet_enc pe on pe.c_fullname = pm.pcori_path + where pm.pcori_path like '\PCORI\ENCOUNTER\ADMITTING_SOURCE\%' + ) +select + obs.encounter_num, codes.pcori_code, + -- Prioritize status: Last OT, UN, NI + decode(codes.pcori_code, 'AF',0,'AL',1,'AV',2,'ED',3,'HH',4,'HO',5,'HS',6,'IP',7, + 'NH',8,'RH',9,'RS',10,'SN',11,'OT',12,'UN',13,'NI',14, 99) as_rank +from blueherondata.observation_fact obs +join codes on codes.concept_cd = obs.concept_cd; + +create index admit_source_enc_rank_idx on admit_source_enc_code_rank(encounter_num, as_rank); +begin +GATHER_TABLE_STATS('admit_source_enc_code_rank'); +end; +/ + +create table admit_source_priority_rank as +select min(as_rank) as_rank, encounter_num +from admit_source_enc_code_rank ranks +group by encounter_num; + +create index admit_source_pr_enc_rank_idx on admit_source_priority_rank(encounter_num, as_rank); +begin +GATHER_TABLE_STATS('admit_source_priority_rank'); +end; +/ + merge into "&&i2b2_data_schema".visit_dimension vd using ( - with codes as ( - select - pm.pcori_path, cd.concept_cd, cd.name_char, - replace(pe.pcori_basecode, 'ADMITTING_SOURCE:', '') pcori_code - from - pcornet_mapping pm - join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' - join "&&i2b2_meta_schema".pcornet_enc pe on pe.c_fullname = pm.pcori_path - where pm.pcori_path like '\PCORI\ENCOUNTER\ADMITTING_SOURCE\%' - ), - enc_as as ( - select obs.encounter_num, codes.pcori_code - from "&&i2b2_data_schema".observation_fact obs - join codes on codes.concept_cd = obs.concept_cd - ), - ranks as ( - select - encounter_num, pcori_code, - -- Prioritize status: Last OT, UN, NI - decode(pcori_code, 'AF',0,'AL',1,'AV',2,'ED',3,'HH',4,'HO',5,'HS',6,'IP',7, - 'NH',8,'RH',9,'RS',10,'SN',11,'OT',12,'UN',13,'NI',14, 99) as_rank - from enc_as - ), - priority_rank as ( - select min(as_rank) as_rank, encounter_num - from ranks - group by encounter_num - ) select distinct ranks.encounter_num, coalesce(ranks.pcori_code, 'NI') pcori_code - from priority_rank pr - join ranks on pr.encounter_num = ranks.encounter_num and pr.as_rank = ranks.as_rank + from admit_source_priority_rank pr + join admit_source_enc_code_rank ranks on pr.encounter_num = ranks.encounter_num + and pr.as_rank = ranks.as_rank ) admt_enc on (admt_enc.encounter_num = vd.encounter_num) when matched then update set vd.admitting_source = admt_enc.pcori_code From 6b5fdeba70589b64b8be1d334c90b5c299545e59 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 10 Aug 2016 11:48:33 -0500 Subject: [PATCH 151/507] Use the fact table primary key to join prescribing table with basis, freq, etc. - Thanks to Russ for pointing out that we may have many medication facts per encounter/concept code. --- Oracle/PCORNetLoader_ora.sql | 78 ++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 21 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index c07e523..72a85f2 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1441,33 +1441,53 @@ drop table supply; create table basis ( pcori_basecode varchar2(50 byte), - c_fullname varchar2(700 byte), - encounter_num number(38,0), - concept_cd varchar2(50 byte) + c_fullname varchar2(700 byte), + encounter_num number(38,0), + concept_cd varchar2(50 byte), + instance_num number(18,0), + start_date date, + provider_id varchar2(50 byte), + modifier_cd varchar2(100 byte) ) ; create table freq ( pcori_basecode varchar2(50 byte), - encounter_num number(38,0), - concept_cd varchar2(50 byte) + encounter_num number(38,0), + concept_cd varchar2(50 byte), + instance_num number(18,0), + start_date date, + provider_id varchar2(50 byte), + modifier_cd varchar2(100 byte) ); create table quantity( nval_num number(18,5), - encounter_num number(38,0), - concept_cd varchar2(50 byte) + encounter_num number(38,0), + concept_cd varchar2(50 byte), + instance_num number(18,0), + start_date date, + provider_id varchar2(50 byte), + modifier_cd varchar2(100 byte) ); create table refills( nval_num number(18,5), - encounter_num number(38,0), - concept_cd varchar2(50 byte) + encounter_num number(38,0), + concept_cd varchar2(50 byte), + instance_num number(18,0), + start_date date, + provider_id varchar2(50 byte), + modifier_cd varchar2(100 byte) ); create table supply( nval_num number(18,5), - encounter_num number(38,0), - concept_cd varchar2(50 byte) + encounter_num number(38,0), + concept_cd varchar2(50 byte), + instance_num number(18,0), + start_date date, + provider_id varchar2(50 byte), + modifier_cd varchar2(100 byte) ); whenever sqlerror exit; @@ -1479,33 +1499,34 @@ begin PMN_DROPSQL('drop index prescribing_patid'); PMN_DROPSQL('drop index prescribing_encounterid'); + PMN_DROPSQL('DROP TABLE basis'); sqltext := 'create table basis as '|| -'(select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2medfact basis '|| +'(select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact basis '|| ' inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num '|| ' join pcornet_med basiscode '|| ' on basis.modifier_cd = basiscode.c_basecode '|| ' and basiscode.c_fullname like ''\PCORI_MOD\RX_BASIS\%'') '; PMN_EXECUATESQL(sqltext); -execute immediate 'create index basis_idx on basis (encounter_num, concept_cd)'; +execute immediate 'create unique index basis_idx on basis (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('BASIS'); PMN_DROPSQL('DROP TABLE freq'); sqltext := 'create table freq as '|| -'(select pcori_basecode,encounter_num,concept_cd from i2b2medfact freq '|| +'(select pcori_basecode,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact freq '|| ' inner join encounter enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num '|| ' join pcornet_med freqcode '|| ' on freq.modifier_cd = freqcode.c_basecode '|| ' and freqcode.c_fullname like ''\PCORI_MOD\RX_FREQUENCY\%'') '; PMN_EXECUATESQL(sqltext); -execute immediate 'create index freq_idx on freq (encounter_num, concept_cd)'; +execute immediate 'create unique index freq_idx on freq (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('FREQ'); PMN_DROPSQL('DROP TABLE quantity'); sqltext := 'create table quantity as '|| -'(select nval_num,encounter_num,concept_cd from i2b2medfact quantity '|| +'(select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact quantity '|| ' inner join encounter enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num '|| ' join pcornet_med quantitycode '|| ' on quantity.modifier_cd = quantitycode.c_basecode '|| @@ -1513,31 +1534,31 @@ sqltext := 'create table quantity as '|| PMN_EXECUATESQL(sqltext); -execute immediate 'create index quantity_idx on quantity (encounter_num, concept_cd)'; +execute immediate 'create unique index quantity_idx on quantity (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('QUANTITY'); PMN_DROPSQL('DROP TABLE refills'); sqltext := 'create table refills as '|| -'(select nval_num,encounter_num,concept_cd from i2b2medfact refills '|| +'(select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact refills '|| ' inner join encounter enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num '|| ' join pcornet_med refillscode '|| ' on refills.modifier_cd = refillscode.c_basecode '|| ' and refillscode.c_fullname like ''\PCORI_MOD\RX_REFILLS\'') '; PMN_EXECUATESQL(sqltext); -execute immediate 'create index refills_idx on refills (encounter_num, concept_cd)'; +execute immediate 'create unique index refills_idx on refills (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('REFILLS'); PMN_DROPSQL('DROP TABLE supply'); sqltext := 'create table supply as '|| -'(select nval_num,encounter_num,concept_cd from i2b2medfact supply '|| +'(select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact supply '|| ' inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| ' join pcornet_med supplycode '|| ' on supply.modifier_cd = supplycode.c_basecode '|| ' and supplycode.c_fullname like ''\PCORI_MOD\RX_DAYS_SUPPLY\'') '; PMN_EXECUATESQL(sqltext); -execute immediate 'create index supply_idx on supply (encounter_num, concept_cd)'; +execute immediate 'create unique index supply_idx on supply (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('SUPPLY'); -- insert data with outer joins to ensure all records are included even if some data elements are missing @@ -1569,22 +1590,37 @@ inner join encounter enc on enc.encounterid = m.encounter_Num left join basis on m.encounter_num = basis.encounter_num and m.concept_cd = basis.concept_Cd + and m.start_date = basis.start_date + and m.provider_id = basis.provider_id + and m.modifier_cd = basis.modifier_cd left join freq on m.encounter_num = freq.encounter_num and m.concept_cd = freq.concept_Cd + and m.start_date = freq.start_date + and m.provider_id = freq.provider_id + and m.modifier_cd = freq.modifier_cd left join quantity on m.encounter_num = quantity.encounter_num and m.concept_cd = quantity.concept_Cd + and m.start_date = quantity.start_date + and m.provider_id = quantity.provider_id + and m.modifier_cd = quantity.modifier_cd left join refills on m.encounter_num = refills.encounter_num and m.concept_cd = refills.concept_Cd + and m.start_date = refills.start_date + and m.provider_id = refills.provider_id + and m.modifier_cd = refills.modifier_cd left join supply on m.encounter_num = supply.encounter_num and m.concept_cd = supply.concept_Cd + and m.start_date = supply.start_date + and m.provider_id = supply.provider_id + and m.modifier_cd = supply.modifier_cd where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); From 4c626b44fedf6e53274cd701c9def9aa835f49ad Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Thu, 18 Aug 2016 15:54:50 -0500 Subject: [PATCH 152/507] populate CONDITION table from ACTIVE problems etc. Our current full list of clinical modifiers is: \Diagnosis\Clinical\ \Diagnosis\Clinical\HOSPITAL_PL_YN\ \Diagnosis\Clinical\MEDICAL_HISTORY_DX\ \Diagnosis\Clinical\PAT_ENC_DX\ \Diagnosis\Clinical\PRIMARY_DX_YN\ \Diagnosis\Clinical\PRINCIPAL_PL_YN\ \Diagnosis\Clinical\PROBLEM_LIST\ \Diagnosis\STATUS_C\ACTIVE\ \Diagnosis\STATUS_C\DELETED\ \Diagnosis\STATUS_C\RESOLVED\ Active is obviously on target for SOURCE=HC "Healthcare problem list"; I'm reasonably confided that MEDICAL_HISTORY_DX, DELETED, and RESOLVED don't belong. We only add HOSPITAL_PL_YN and PRINCIPAL_PL_YN modifiers when the YN is Y, so I included those along with PAT_ENC_DX. I waffled on PRINCIPAL_PL_YN and left it out thinking it might match irrelevant stuff. I think it wouldn't matter: I think we only add a PRINCIPAL_PL_YN modifier if we also add one of the others above. --- Oracle/pcornet_mapping.csv | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 596f9cf..044a7ab 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -55,6 +55,10 @@ pcori_path,local_path \PCORI_MOD\PDX\P\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\discharge_primary\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\DI\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\AD\,\Diagnosis\\modifier_dx\billing_diagnosis\admit\ +\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\Clinical\HOSPITAL_PL_YN\ +\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\Clinical\PAT_ENC_DX\ +\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\Clinical\PRINCIPAL_PL_YN\ +\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\STATUS_C\ACTIVE\ \PCORI\DEMOGRAPHIC\BIOBANK_FLAG\Y\,\i2b2\Specimens\ \PCORI\DEMOGRAPHIC\HISPANIC\N\,"\i2b2\Demographics\Ethnicity\Non Hispanic, Latino or Spanish Origin\" \PCORI\DEMOGRAPHIC\HISPANIC\NI\,\i2b2\Demographics\Ethnicity\Not Recorded\ From a45991614699f2dc896c420e41a4e9b27639533d Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 9 Sep 2016 12:08:55 -0500 Subject: [PATCH 153/507] un-comment PCORNetCondition --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 72a85f2..f237736 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1795,7 +1795,7 @@ begin PCORNetDemographic; PCORNetEncounter; PCORNetDiagnosis; --- TODO: Put this back - avoid performance issues for now: PCORNetCondition; +PCORNetCondition; PCORNetProcedure; PCORNetVital; PCORNetEnroll; From 13f76e52785afea81c79cddb6439e63e2ceae0fb Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 9 Sep 2016 14:55:59 -0500 Subject: [PATCH 154/507] include medical history diagnoses as PR=patient-reported --- Oracle/pcornet_mapping.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 044a7ab..c8d330f 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -59,6 +59,7 @@ pcori_path,local_path \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\Clinical\PAT_ENC_DX\ \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\Clinical\PRINCIPAL_PL_YN\ \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\STATUS_C\ACTIVE\ +\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\PR\,\Diagnosis\Clinical\MEDICAL_HISTORY_DX\ \PCORI\DEMOGRAPHIC\BIOBANK_FLAG\Y\,\i2b2\Specimens\ \PCORI\DEMOGRAPHIC\HISPANIC\N\,"\i2b2\Demographics\Ethnicity\Non Hispanic, Latino or Spanish Origin\" \PCORI\DEMOGRAPHIC\HISPANIC\NI\,\i2b2\Demographics\Ethnicity\Not Recorded\ From 1a15b033b0ba4178ed56ff898a116484af89bf50 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 9 Sep 2016 15:03:44 -0500 Subject: [PATCH 155/507] include NTDS diagnoses in CONDITION as RG=Registry cohort --- Oracle/pcornet_mapping.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index c8d330f..0411bf9 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -60,6 +60,7 @@ pcori_path,local_path \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\Clinical\PRINCIPAL_PL_YN\ \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\STATUS_C\ACTIVE\ \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\PR\,\Diagnosis\Clinical\MEDICAL_HISTORY_DX\ +\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\RG\,\Diagnosis\NTDS\ \PCORI\DEMOGRAPHIC\BIOBANK_FLAG\Y\,\i2b2\Specimens\ \PCORI\DEMOGRAPHIC\HISPANIC\N\,"\i2b2\Demographics\Ethnicity\Non Hispanic, Latino or Spanish Origin\" \PCORI\DEMOGRAPHIC\HISPANIC\NI\,\i2b2\Demographics\Ethnicity\Not Recorded\ From 648e6f60b1feb0e4e1a7e93a00659903defbd5d0 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 9 Sep 2016 14:55:59 -0500 Subject: [PATCH 156/507] include medical history diagnoses as PR=patient-reported --- Oracle/pcornet_mapping.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 044a7ab..c8d330f 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -59,6 +59,7 @@ pcori_path,local_path \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\Clinical\PAT_ENC_DX\ \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\Clinical\PRINCIPAL_PL_YN\ \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\STATUS_C\ACTIVE\ +\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\PR\,\Diagnosis\Clinical\MEDICAL_HISTORY_DX\ \PCORI\DEMOGRAPHIC\BIOBANK_FLAG\Y\,\i2b2\Specimens\ \PCORI\DEMOGRAPHIC\HISPANIC\N\,"\i2b2\Demographics\Ethnicity\Non Hispanic, Latino or Spanish Origin\" \PCORI\DEMOGRAPHIC\HISPANIC\NI\,\i2b2\Demographics\Ethnicity\Not Recorded\ From 5579c32de18cfc3d798a8c7fd90be2c3258eb317 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 9 Sep 2016 15:03:44 -0500 Subject: [PATCH 157/507] include NTDS diagnoses in CONDITION as RG=Registry cohort --- Oracle/pcornet_mapping.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index c8d330f..0411bf9 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -60,6 +60,7 @@ pcori_path,local_path \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\Clinical\PRINCIPAL_PL_YN\ \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\STATUS_C\ACTIVE\ \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\PR\,\Diagnosis\Clinical\MEDICAL_HISTORY_DX\ +\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\RG\,\Diagnosis\NTDS\ \PCORI\DEMOGRAPHIC\BIOBANK_FLAG\Y\,\i2b2\Specimens\ \PCORI\DEMOGRAPHIC\HISPANIC\N\,"\i2b2\Demographics\Ethnicity\Non Hispanic, Latino or Spanish Origin\" \PCORI\DEMOGRAPHIC\HISPANIC\NI\,\i2b2\Demographics\Ethnicity\Not Recorded\ From f487ca78c41ff56b738e7d2d5fa7ff8c423a1c87 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 19 Oct 2016 17:10:05 -0500 Subject: [PATCH 158/507] Tests to show if height/weight are missing from vital. --- Oracle/cdm_transform_tests.sql | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index ff8374f..7cf8cc0 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -25,6 +25,44 @@ comment on column test_cases.distinct_patid_n is 'patient count'; comment on column test_cases.pass is '1 for pass, 0 for fail'; +/* Test that we have some height/weight measurements +*/ +insert into test_cases (query_name, description, pass, obs, record_n, record_pct) +with total_vital as ( + select count(*) qty from pcornet_cdm.vital + ), +ht as ( + select count(*) qty from pcornet_cdm.vital + where ht is not null + ), +wt as ( + select count(*) qty from pcornet_cdm.vital + where wt is not null + ), +results as ( + select + round((ht.qty/total_vital.qty) * 100) pct_ht, + round((wt.qty/total_vital.qty) * 100) pct_wt + from total_vital cross join ht cross join wt + ) +select 'some_height_measurements_4335' query_name + , 'Make sure we have at least some height records' description + , case when ht.qty > 0 then 1 else 0 end pass + , rownum obs + , ht.qty record_n + , results.pct_ht record_pct +from total_vital cross join ht cross join results +union all +select 'some_weight_measurements_4335' query_name + , 'Make sure we have at least some weight records' description + , case when wt.qty > 0 then 1 else 0 end pass + , rownum obs + , wt.qty record_n + , results.pct_wt record_pct +from total_vital cross join wt cross join results +; +commit; + /* Test to make sure we got about the same number of patients in the CDM diagnoses that we do in i2b2. */ From 051bac8c3983ad72a9357888ac232a08d32203f9 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 19 Oct 2016 17:47:34 -0500 Subject: [PATCH 159/507] Use visit details rather than flowsheets for height and weight - Weight is in ounces not kg in visit vitals. Thanks to Russ for pointing out the correct paths to use. --- Oracle/cdm_postproc.sql | 6 +++--- Oracle/pcornet_mapping.csv | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Oracle/cdm_postproc.sql b/Oracle/cdm_postproc.sql index 9e90267..9324c2d 100644 --- a/Oracle/cdm_postproc.sql +++ b/Oracle/cdm_postproc.sql @@ -20,10 +20,10 @@ using encounter e on (p.encounterid = e.encounterid) when matched then update set p.rx_providerid = e.providerid; -/* Currently in HERON, we have hight in cm and weight in kg - the CDM wants -height in inches and weight in pounds. */ +/* Currently in HERON, we have hight in cm and weight in oz (from visit vitals). +The CDM wants height in inches and weight in pounds. */ update vital v set v.ht = v.ht / 2.54; -update vital v set v.wt = v.wt * 2.20462; +update vital v set v.wt = v.wt / 16; /* Populate death table. Eventually, we expect this to be added to the upstream transform code. diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 0411bf9..1eeb9c3 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -89,11 +89,11 @@ pcori_path,local_path \PCORI\DIAGNOSIS\DX_TYPE\09\,\i2b2\Diagnoses\ \PCORI\PROCEDURE\PX_TYPE\C4\,\i2b2\Procedures\PRC\Metathesaurus CPT Hierarchical Terms\ \PCORI\PROCEDURE\PX_TYPE\09\,\i2b2\Procedures\PRC\ICD9 (Inpatient)\ -\PCORI\VITAL\HT\,\i2b2\Flowsheet\KU\GEN CARD\IP/OP HEIGHT/WEIGHT T:900445\KU GEN CARD G IP/OP PROCEDURE HEIGHT/WEI G:900612 I#1\HEIGHT M:11 I#1\ +\PCORI\VITAL\HT\,\i2b2\Visit Details\Vitals\HEIGHT\ \PCORI\VITAL\BP\DIASTOLIC\,\i2b2\Flowsheet\KU\GEN TMP CARD ECHO VITALS T:900426\KU GEN CARD GRP CV COMMON VITALS G:910000 I#2\BLOOD PRESSURE M:5 I#2\DIASTOLIC\ \PCORI\VITAL\BP\SYSTOLIC\,\i2b2\Flowsheet\KU\GEN TMP CARD ECHO VITALS T:900426\KU GEN CARD GRP CV COMMON VITALS G:910000 I#2\BLOOD PRESSURE M:5 I#2\SYSTOLIC\ \PCORI\VITAL\ORIGINAL_BMI\,\i2b2\Flowsheet\MODEL\VITAL SIGNS COMPLEX T:31010\KU IP GRP ADULT HEIGHT/WEIGHT G:300220 I#4\KU IP ROW BMI M:301070 I#6\ -\PCORI\VITAL\WT\,\i2b2\Flowsheet\KU\GEN CARD\IP/OP HEIGHT/WEIGHT T:900445\KU GEN CARD G IP/OP PROCEDURE HEIGHT/WEI G:900612 I#1\WEIGHT/SCALE M:14 I#2\ +\PCORI\VITAL\WT\,\i2b2\Visit Details\Vitals\WEIGHT\ \PCORI\VITAL\TOBACCO\SMOKING\YES\01\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Current Every Day Smoker\ \PCORI\VITAL\TOBACCO\SMOKING\YES\02\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Current Some Day Smoker\ \PCORI\VITAL\TOBACCO\SMOKING\03\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Former Smoker\ From 3d533865a5b7a8db50e0f0f8ded325a6ec448f04 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 31 Oct 2016 17:02:01 -0500 Subject: [PATCH 160/507] Report basic CDM statistics. --- Oracle/cdm_stats.sql | 45 +++++++++++++++++++++++++++++++++++++ Oracle/run-i2p-transform.sh | 6 +++++ 2 files changed, 51 insertions(+) create mode 100644 Oracle/cdm_stats.sql diff --git a/Oracle/cdm_stats.sql b/Oracle/cdm_stats.sql new file mode 100644 index 0000000..a6b903d --- /dev/null +++ b/Oracle/cdm_stats.sql @@ -0,0 +1,45 @@ +select 'DEMOGRAPHIC' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", 'N/A' "Distinct Encounters" from DEMOGRAPHIC + union all +select 'ENROLLMENT' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", 'N/A' "Distinct Encounters" from ENROLLMENT + union all +select 'ENCOUNTER' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from ENCOUNTER + union all +select 'DIAGNOSIS' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from DIAGNOSIS + union all +select 'PROCEDURES' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from PROCEDURES + union all +select 'VITAL' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from VITAL + union all +select 'DISPENSING' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", 'N/A' "Distinct Encounters" from DISPENSING + union all +select 'LAB_RESULT_CM' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from LAB_RESULT_CM + union all +select 'CONDITION' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from CONDITION + union all +select 'PRO_CM' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from PRO_CM + union all +select 'PRESCRIBING' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from PRESCRIBING + union all +select 'PCORNET_TRIAL' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", 'N/A' "Distinct Encounters" from PCORNET_TRIAL + union all +select 'DEATH' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", 'N/A' "Distinct Encounters" from DEATH + union all +select 'DEATH_CAUSE' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", 'N/A' "Distinct Encounters" from DEATH_CAUSE + union all +select 'HARVEST' "Table", to_char(count(*)) "Total Rows", 'N/A' "Distinct Patients", 'N/A' "Distinct Encounters" from HARVEST +; + +select sum("Size (MB)") "Total size, all tables (MB)" from ( + select + segment_name "Table", bytes/1024/1024 "Size (MB)" + from dba_segments + where owner = '&&pcornet_cdm_user' + and segment_type = 'TABLE' + and segment_name in ('DEMOGRAPHIC', 'ENROLLMENT', 'ENCOUNTER', 'DIAGNOSIS', + 'PROCEDURES', 'VITAL', 'DISPENSING', 'LAB_RESULT_CM', 'CONDITION', 'PRO_CM', + 'PRESCRIBING', 'PCORNET_TRIAL', 'DEATH', 'DEATH_CAUSE', 'HARVEST') + ); + +SELECT sum(bytes/1024/1024/1024) "Total schema size (GB)" + FROM dba_segments + WHERE owner = '&&pcornet_cdm_user'; diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index e6ff831..2817345 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -68,6 +68,8 @@ connect ${pcornet_cdm_user}/${pcornet_cdm} set echo on; set timing on; +set linesize 3000; +set pagesize 5000; WHENEVER SQLERROR CONTINUE; @@ -100,6 +102,7 @@ define terms_table=${terms_table} define min_pat_list_date_dd_mon_rrrr=${min_pat_list_date_dd_mon_rrrr} define min_visit_date_dd_mon_rrrr=${min_visit_date_dd_mon_rrrr} define enrollment_months_back=${enrollment_months_back} +define pcornet_cdm_user=${pcornet_cdm_user} -- Local terminology mapping start pcornet_mapping.sql @@ -113,6 +116,9 @@ start PCORNetLoader_ora.sql -- Post-process steps start cdm_postproc.sql +-- Report stats +start cdm_stats.sql + -- CDM transform tests start cdm_transform_tests.sql From ed1298507d6e88d2aa9254a3f3be09cde6fd2619 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 1 Nov 2016 09:47:35 -0500 Subject: [PATCH 161/507] PDX is "relevant only on IP and IS encounters" (#4147) --- Oracle/PCORNetLoader_ora.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index f237736..fba2770 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -993,7 +993,8 @@ sqltext := 'insert into diagnosis (patid, encounterid, enc_type, admit_date, p 'select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, factline.start_date, factline.provider_id, diag.pcori_basecode, '|| 'SUBSTR(diag.c_fullname,18,2) dxtype, '|| ' CASE WHEN enc_type=''AV'' THEN ''FI'' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,'':'')+1,2) ,''NI'')END, '|| -' nvl(SUBSTR(pdxsource,INSTR(pdxsource, '':'')+1,2),''NI'') '|| -- jgk bugfix 9/28/15 +' CASE WHEN enc_type in (''ED'', ''AV'', ''OA'') THEN ''X'' -- PDX is "relevant only on IP and IS encounters" + ELSE nvl(SUBSTR(pdxsource,INSTR(pdxsource, '':'')+1,2),''NI'') END '|| 'from i2b2fact factline '|| 'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| ' left outer join sourcefact '|| From 93b887731ee9dad7c5ab271db1e0ba7f7ef20bb5 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Tue, 1 Nov 2016 14:22:49 -0500 Subject: [PATCH 162/507] Log constraint violations when inserting into condition (#4270). --- Oracle/PCORNetLoader_ora.sql | 25 ++++++++++++++++++++++++- Oracle/cdm_transform_tests.sql | 17 +++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index f237736..85b78ed 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -69,6 +69,25 @@ BEGIN END PMN_ExecuateSQL; / +--ACK: http://dba.stackexchange.com/questions/9441/how-to-catch-and-handle-only-specific-oracle-exceptions +create or replace procedure create_error_table(table_name varchar2) as +sqltext varchar2(4000); + +begin + dbms_errlog.create_error_log(dml_table_name => table_name); +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -955 THEN + NULL; -- suppresses ORA-00955 exception ("name is already used by an existing object") + ELSE + RAISE; + END IF; +-- Delete rows from a previous run in case the table already existed +sqltext := 'delete from ERR$_' || table_name; +PMN_Execuatesql(sqltext); +end; +/ + @@ -1042,6 +1061,8 @@ sqltext := 'create table sourcefact2 as '|| 'where dxsource.c_fullname like ''\PCORI_MOD\CONDITION_OR_DX\%'''; PMN_EXECUATESQL(sqltext); +create_error_table('CONDITION'); + sqltext := 'insert into condition (patid, encounterid, report_date, resolve_date, condition, condition_type, condition_status, condition_source) '|| 'select distinct factline.patient_num, min(factline.encounter_num) encounterid, min(factline.start_date) report_date, NVL(max(factline.end_date),null) resolve_date, diag.pcori_basecode, '|| 'SUBSTR(diag.c_fullname,18,2) condition_type, '|| @@ -1058,7 +1079,9 @@ sqltext := 'insert into condition (patid, encounterid, report_date, resolve_date 'and factline.start_date=sf.start_Date '|| 'where diag.c_fullname like ''\PCORI\DIAGNOSIS\%'' '|| 'and sf.c_fullname like ''\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\%'' '|| -'group by factline.patient_num, diag.pcori_basecode, diag.c_fullname '; +'group by factline.patient_num, diag.pcori_basecode, diag.c_fullname ' || +'log errors into ERR$_CONDITION reject limit unlimited' +; PMN_EXECUATESQL(sqltext); diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 7cf8cc0..d832582 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -24,6 +24,23 @@ comment on column test_cases.record_pct is 'percent of all observations'; comment on column test_cases.distinct_patid_n is 'patient count'; comment on column test_cases.pass is '1 for pass, 0 for fail'; +/* Report how many condition table rows were rejected. +*/ +insert into test_cases (query_name, description, pass, obs, record_n, record_pct) +with err as ( + select count(*) qty from pcornet_cdm.ERR$_CONDITION + ), +total_condition as ( + select count(*) qty from pcornet_cdm.condition + ) +select 'rejected_condition_rows' query_name + , 'Report how many rows were rejected from the condition table due to table constraints.' description + , case when err.qty < 1000 then 1 else 0 end pass + , rownum obs + , err.qty record_n + , (err.qty / tc.qty) * 100 record_pct +from total_condition tc cross join err +; /* Test that we have some height/weight measurements */ From 1c0c19a6f4424e7b01a4c21a1e1e6d64a5a0cc91 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 2 Nov 2016 09:16:09 -0500 Subject: [PATCH 163/507] for PDX, use whitelist of ENC_TYPE rather than blacklist --- Oracle/PCORNetLoader_ora.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index fba2770..74eebbb 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -993,8 +993,9 @@ sqltext := 'insert into diagnosis (patid, encounterid, enc_type, admit_date, p 'select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, factline.start_date, factline.provider_id, diag.pcori_basecode, '|| 'SUBSTR(diag.c_fullname,18,2) dxtype, '|| ' CASE WHEN enc_type=''AV'' THEN ''FI'' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,'':'')+1,2) ,''NI'')END, '|| -' CASE WHEN enc_type in (''ED'', ''AV'', ''OA'') THEN ''X'' -- PDX is "relevant only on IP and IS encounters" - ELSE nvl(SUBSTR(pdxsource,INSTR(pdxsource, '':'')+1,2),''NI'') END '|| +' CASE WHEN enc_type in (''IP'', ''IS'') -- PDX is "relevant only on IP and IS encounters"'|| +' THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, '':'')+1,2),''NI'')'|| +' ELSE ''X'' END '|| 'from i2b2fact factline '|| 'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| ' left outer join sourcefact '|| From 01ff270f74b6c1db0c434b72336909b5f3328e46 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 2 Nov 2016 12:16:27 -0500 Subject: [PATCH 164/507] Fix SQL quoting typo (#4147) --- Oracle/PCORNetLoader_ora.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 74eebbb..0a1ecb4 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -992,10 +992,10 @@ GATHER_TABLE_STATS('PDXFACT'); sqltext := 'insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, pdx) '|| 'select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, factline.start_date, factline.provider_id, diag.pcori_basecode, '|| 'SUBSTR(diag.c_fullname,18,2) dxtype, '|| -' CASE WHEN enc_type=''AV'' THEN ''FI'' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,'':'')+1,2) ,''NI'')END, '|| -' CASE WHEN enc_type in (''IP'', ''IS'') -- PDX is "relevant only on IP and IS encounters"'|| -' THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, '':'')+1,2),''NI'')'|| -' ELSE ''X'' END '|| +' CASE WHEN enc_type=''AV'' THEN ''FI'' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,'':'')+1,2) ,''NI'') END, '|| +' CASE WHEN enc_type in (''IP'', ''IS'') -- PDX is "relevant only on IP and IS encounters" + THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, '':'')+1,2),''NI'') + ELSE ''X'' END '|| 'from i2b2fact factline '|| 'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| ' left outer join sourcefact '|| From 6666c9c637b2a60b62d139dcc4aacd0934004a24 Mon Sep 17 00:00:00 2001 From: Matt Hoag Date: Thu, 10 Nov 2016 15:12:27 -0600 Subject: [PATCH 165/507] =?UTF-8?q?Assign=20a=20single=20encounter=5Ftype?= =?UTF-8?q?=20based=20on=20all=20relevant=20encounter=20types.=20=20-=20In?= =?UTF-8?q?stead=20of=20just=20=E2=80=9Cmax=E2=80=9D,=20employ=20pivot=20t?= =?UTF-8?q?o=20find=20all=20encounter=20types=20=20=20=20associated=20with?= =?UTF-8?q?=20each=20encounter=20and=20deterministically=20and=20=20=20=20?= =?UTF-8?q?intentionally=20coerce=20those=20types=20into=20a=20single=20en?= =?UTF-8?q?counter=20type.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Oracle/heron_encounter_style.sql | 36 +++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index fffe407..58e0489 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -154,13 +154,35 @@ using ( join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' where pm.pcori_path like '\PCORI\ENCOUNTER\ENC_TYPE\%' ) --- Note: our patient-day style encounters etc. may result in --- multiple encounter types for a single encounter. -select obs.encounter_num, max(et.pcori_code) pcori_code -from "&&i2b2_data_schema".observation_fact obs -join enc_type_codes et on et.concept_cd = obs.concept_cd -group by obs.encounter_num -) et on ( vd.encounter_num = et.encounter_num ) + /* Note: our patient-day style encounters etc. may result in + multiple encounter types for a single encounter. + Therefore we need to pivot to all the encounter types associated with each + encounter and then choose (intentionally and deterministically) the + single encounter type that will prevail for each encounter + */ + , obs_enc_type_pivot as ( + select * + from (select obs.encounter_num, et.pcori_code + from "&&i2b2_data_schema".observation_fact obs + join enc_type_codes et on et.concept_cd = obs.concept_cd) + pivot + ( + max(pcori_code) + for pcori_code in ('EI' as ei, 'IP' as ip, 'ED' as ed, 'IS' as e_is + , 'AV' as av, 'OA' as oa, 'OT' as ot + , 'NI' as ni, 'UN' as un) + ) + ) + /* Single Encounter type to Encounter coercion: + IP + ED = EI + EI > IP || ED > IS > AV > OA > OT > NI > UN + */ + select encounter_num, + case when ip is not null and ed is not null then 'EI' + else coalesce(ei, ip, ed, e_is, av, oa, ot, ni, un, 'UN') + end as pcori_code + from obs_enc_type_pivot) et +on ( vd.encounter_num = et.encounter_num ) when matched then update set inout_cd = et.pcori_code; -- 11,085,911 rows merged. From 425b5f7a7c42bbe42e9a75235d397250cd2bcdb0 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 16 Nov 2016 12:22:25 -0600 Subject: [PATCH 166/507] IDX diagnoses were not mapped to DIAGNOSIS table Our `diag_pat_count_ok` test has been failing for some time: `select count(distinct patid) qty from diagnosis` is significantly smaller than the corresponding query from `i2b2fact` and `concept_dimension`. This query showed that a lot of IDX data isn't mapped to CDM due to missing modifier mappings: ```sql -- How many i2b2 diagnosis observations are mapped to which PCORNET modifiers? select count(*), obs.modifier_cd, dxsource.c_name, dxsource.c_fullname from i2b2fact obs left join ( select * from pcornet_diag where c_fullname like '\PCORI_MOD\CONDITION_OR_DX\%' ) dxsource on dxsource.c_basecode = obs.modifier_cd where obs.concept_cd in ( select concept_cd from blueherondata.concept_dimension cd where concept_path like '\i2b2\Diagnoses\%' ) group by obs.modifier_cd, dxsource.c_name, dxsource.c_fullname ; ``` And this lets us review our diagnosis modifier mappings: ```sql -- Which diagnosis modifiers are mapped and which are not? select md.name_char, md.modifier_cd, pmap.*, md.modifier_path from blueherondata.modifier_dimension md left join pcornet_mapping pmap on md.modifier_path like (pmap.local_path || '%') where md.modifier_path like '\Diagnosis\%' and md.modifier_cd is not null ; ``` --- Oracle/pcornet_mapping.csv | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 1eeb9c3..b597f7a 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -55,6 +55,11 @@ pcori_path,local_path \PCORI_MOD\PDX\P\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\discharge_primary\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\DI\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\AD\,\Diagnosis\\modifier_dx\billing_diagnosis\admit\ +"\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\UN\","\Diagnosis\Billing\UHC_DIAGNOSIS\" +"\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\FI\","\Diagnosis\Billing\UKP\" +"\PCORI_MOD\PDX\X\","\Diagnosis\Billing\Primary\" +"\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\FI\","\Diagnosis\\modifier_dx\billing_diagnosis\professional\" +"\PCORI_MOD\PDX\X\","\Diagnosis\\modifier_dx\billing_diagnosis\professional\professional_primary\" \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\Clinical\HOSPITAL_PL_YN\ \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\Clinical\PAT_ENC_DX\ \PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\HC\,\Diagnosis\Clinical\PRINCIPAL_PL_YN\ From 9d2b1466a496a50f38ff6a87e8f22ca80d886377 Mon Sep 17 00:00:00 2001 From: Matt Hoag Date: Mon, 21 Nov 2016 11:11:13 -0600 Subject: [PATCH 167/507] Added ICD10 Procedures to `pcornet_proc` #4284 - Also condensed ICD9, ICD10, and CPT procedures to a single `delete` and `insert` statement. --- Oracle/pcornet_mapping.sql | 49 ++++++++++++-------------------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index df73fb5..3af0446 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -170,52 +170,33 @@ join "&&i2b2_meta_schema"."&&terms_table" i2b2 on i2b2.c_fullname like pcornet_m commit; ---select * -delete -from "&&i2b2_meta_schema".PCORNET_PROC -where c_fullname like '\PCORI\PROCEDURE\09\_%' +-- TODO may be better represented in pcornet_mapping.csv +create or replace view proc_local_to_pcori as +select '\PCORI\PROCEDURE\09\' as pcori_path, '\i2b2\Procedures\PRC\ICD9 (Inpatient)\' as local_path from dual +union all select '\PCORI\PROCEDURE\10\' as pcori_path, '\i2b2\Procedures\ICD10\' as local_path from dual +union all select '\PCORI\PROCEDURE\C4\' as pcori_path, '\i2b2\Procedures\PRC\Metathesaurus CPT Hierarchical Terms\' as local_path from dual ; -/* Replace PCORNet Procedure (ICD9) hierarchy with the local hierarchy. -*/ -insert into "&&i2b2_meta_schema".PCORNET_PROC -select - ht.c_hlevel, - replace(ht.c_fullname, '\i2b2\Procedures\PRC\ICD9 (Inpatient)', '\PCORI\PROCEDURE\09') c_fullname, - ht.c_name, ht.c_synonym_cd, ht.c_visualattributes, - ht.c_totalnum, ht.c_basecode, ht.c_metadataxml, ht.c_facttablecolumn, ht.c_tablename, - ht.c_columnname, ht.c_columndatatype, ht.c_operator, ht.c_dimcode, ht.c_comment, - ht.c_tooltip, ht.m_applied_path, ht.update_date, ht.download_date, ht.import_date, - ht.sourcesystem_cd, ht.valuetype_cd, ht.m_exclusion_cd, ht.c_path, ht.c_symbol, - ht.c_basecode pcori_basecode -from - "&&i2b2_meta_schema"."&&terms_table" ht -where c_fullname like '\i2b2\Procedures\PRC\ICD9 (Inpatient)\_%' order by c_hlevel -; - - --select * -delete -from "&&i2b2_meta_schema".PCORNET_PROC -where c_fullname like '\PCORI\PROCEDURE\C4\%' -; +delete +from "&&i2b2_meta_schema".PCORNET_PROC p_proc +where exists (select 1 + from proc_local_to_pcori m + where p_proc.c_fullname like m.pcori_path || '%'); -/* Replace PCORNet Procedure (ICD9) hierarchy with the local hierarchy. -*/ insert into "&&i2b2_meta_schema".PCORNET_PROC select ht.c_hlevel, - replace(ht.c_fullname, '\i2b2\Procedures\PRC\Metathesaurus CPT Hierarchical Terms', '\PCORI\PROCEDURE\C4') c_fullname, + replace(ht.c_fullname, m.local_path, m.pcori_path) c_fullname, ht.c_name, ht.c_synonym_cd, ht.c_visualattributes, ht.c_totalnum, ht.c_basecode, ht.c_metadataxml, ht.c_facttablecolumn, ht.c_tablename, ht.c_columnname, ht.c_columndatatype, ht.c_operator, ht.c_dimcode, ht.c_comment, ht.c_tooltip, ht.m_applied_path, ht.update_date, ht.download_date, ht.import_date, ht.sourcesystem_cd, ht.valuetype_cd, ht.m_exclusion_cd, ht.c_path, ht.c_symbol, - ht.c_basecode pcori_basecode -from - "&&i2b2_meta_schema"."&&terms_table" ht -where c_fullname like '\i2b2\Procedures\PRC\Metathesaurus CPT Hierarchical Terms\%' order by c_hlevel -; + ht.c_basecode pcori_basecode +from "&&i2b2_meta_schema"."&&terms_table" ht +join proc_local_to_pcori m on ht.c_fullname like m.local_path || '%' +order by ht.c_hlevel; /* MS-DRGs */ From 72f57634481a0147ce775a9ca461a9f0332e3e99 Mon Sep 17 00:00:00 2001 From: Matt Hoag Date: Mon, 21 Nov 2016 13:21:56 -0600 Subject: [PATCH 168/507] Add ICD10 diagnoses ontology to `pcornet_cdm` #4285 - Followed same pattern as ICD9 --- Oracle/pcornet_mapping.sql | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index df73fb5..5895da9 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -137,6 +137,48 @@ order by c_hlevel ; +/* Replace PCORNet ICD10 diagnoses hierarchy with the local hierarchy filling in +the pcornet_basecode with the expected values. +*/ + +create or replace view icd10_diag_local_to_pcori as +select '\PCORI\DIAGNOSIS\10\' as pcori_path + , '\i2b2\Diagnoses\ICD10\' as local_path + , 'ICD10:' as pcori_scheme + , 'KUH|DX_ID:' as local_scheme +from dual; + +insert into "&&i2b2_meta_schema".PCORNET_DIAG +with terms_dxi as ( + select + cicd.code dxicd, ht.*, m.* + from "&&i2b2_meta_schema"."&&terms_table" ht + join icd10_diag_local_to_pcori m + on ht.c_fullname like m.local_path||'%' + -- TODO: Stop cheating by going back to Clarity + left join clarity.edg_current_icd10@id cicd + on to_char(cicd.dx_id) = replace(ht.c_basecode, m.local_scheme, '') + order by ht.c_hlevel + ) + +select + td.c_hlevel, + replace(td.c_fullname, local_path, pcori_path) c_fullname, + td.c_name, td.c_synonym_cd, td.c_visualattributes, + td.c_totalnum, td.c_basecode, td.c_metadataxml, td.c_facttablecolumn, td.c_tablename, + td.c_columnname, td.c_columndatatype, td.c_operator, td.c_dimcode, td.c_comment, + td.c_tooltip, td.m_applied_path, td.update_date, td.download_date, td.import_date, + td.sourcesystem_cd, td.valuetype_cd, td.m_exclusion_cd, td.c_path, td.c_symbol, + case + when td.dxicd is not null then td.dxicd + when td.c_basecode like pcori_scheme||'%' then replace(td.c_basecode, pcori_scheme, '') + else null + end pcori_basecode +from terms_dxi td +order by c_hlevel +; + + -- Other diagnosis mappings such as modifiers for PDX, DX_SOURCE. insert into "&&i2b2_meta_schema".PCORNET_DIAG SELECT pcornet_diag.C_HLEVEL+1, From 8c4d1ee13779aa9c5132e90fe7a8206ecb41b484 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 28 Nov 2016 14:45:39 -0600 Subject: [PATCH 169/507] Add Surescripts dispensing mapping. --- Oracle/pcornet_mapping.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index b597f7a..d0fc797 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -1,4 +1,5 @@ pcori_path,local_path +\PCORI_MOD\RX_BASIS\DI\,\Medication\Surescripts\ \PCORI_MOD\RX_BASIS\PR\,\Medication\Inpatient\ \PCORI_MOD\RX_BASIS\PR\,\Medication\Other Orders\ \PCORI_MOD\RX_BASIS\PR\,\Medication\Outpatient\ From 460b63be52286e601af153cd5e0fadccef0bed2b Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 29 Nov 2016 11:44:23 -0600 Subject: [PATCH 170/507] Added NDC mapping for DISPENSING. --- Oracle/pcornet_mapping.sql | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index df73fb5..386806e 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -380,7 +380,8 @@ terms_rx as ( "&&i2b2_meta_schema"."&&terms_table" ht left join rxnorm_mapping cm2rx on to_char(cm2rx.clarity_med_id) = replace(ht.c_basecode, 'KUH|MEDICATION_ID:', '') where c_fullname like '\i2b2\Medications%' - and ht.c_visualattributes not like '%H%' + and c_basecode not like 'NDC:%' -- We'll handle NDCs seperately below + and ht.c_visualattributes not like '%H%' ) select rx.c_hlevel + 1 c_hlevel, @@ -404,6 +405,23 @@ select --order by trx.c_hlevel ) rx; +-- Handle mapping NDC codes for DISPENSING +insert into "&&i2b2_meta_schema".PCORNET_MED +select + c_hlevel + 1 c_hlevel, + replace(c_fullname, '\i2b2\Medications\', '\PCORI\MEDICATION\RXNORM_CUI\') c_fullname, + c_name, c_synonym_cd, c_visualattributes, + c_totalnum, c_basecode, c_metadataxml, c_facttablecolumn, c_tablename, + c_columnname, c_columndatatype, c_operator, c_dimcode, c_comment, + c_tooltip, m_applied_path, update_date, download_date, import_date, + sourcesystem_cd, valuetype_cd, m_exclusion_cd, c_path, c_symbol, + null pcori_basecode, null pcori_cui, -- This might not work + substr(c_basecode, 5) pcori_ndc +from BLUEHERONMETADATA.HERON_TERMS +where c_fullname like '\i2b2\Medications%' + and c_basecode like 'NDC:%' +; + delete from "&&i2b2_meta_schema".PCORNET_MED where sourcesystem_cd='MAPPING'; From fbbe59b8daed7567f0534329275ea972a2d3fc43 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 30 Nov 2016 15:25:14 -0600 Subject: [PATCH 171/507] Updated RX_BASIS mappings to address to be more accurate. --- Oracle/pcornet_mapping.csv | 57 ++++---------------------------------- 1 file changed, 5 insertions(+), 52 deletions(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index b597f7a..235a925 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -1,57 +1,10 @@ pcori_path,local_path -\PCORI_MOD\RX_BASIS\PR\,\Medication\Inpatient\ -\PCORI_MOD\RX_BASIS\PR\,\Medication\Other Orders\ -\PCORI_MOD\RX_BASIS\PR\,\Medication\Outpatient\ -\PCORI_MOD\RX_BASIS\PR\,\Medication\PRN Inpatient Order\ +\PCORI_MOD\RX_BASIS\PR\02\,\Medication\Inpatient\ +\PCORI_MOD\RX_BASIS\PR\OT\,\Medication\Other Orders\ +\PCORI_MOD\RX_BASIS\PR\01\,\Medication\Outpatient\ +\PCORI_MOD\RX_BASIS\PR\02\,\Medication\PRN Inpatient Order\ \PCORI_MOD\RX_BASIS\PR\01\,\Medication\Dispensed\ -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Automatically Held\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Bolus from Syringe\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Bolus.\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Bolus\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Cabinet Pull\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Canceled Entry\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Downtime Given - New Bag\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Downtime Given\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Downtime Not Given\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Due\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\ED-Infusing Upon Admit\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Given - Without Order\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Given - Patient Supplied\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Given - Without Order\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Given by another Provider\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Given\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Held\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Infusion Completed\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Infusion Home with Patient\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\MAR Hold\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\MAR Unhold\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Med Not Available\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Med Not Given - Resch\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Missed\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\NPO\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\New Bag\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\PCA Check/Change\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\PHTN Order Verified\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Patch Applied\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Patch Removed\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Patient/Family Admin\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Paused\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Peak/Trough\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Pending\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Per Protocol\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Piggyback Rate Change\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Push\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Rate Change\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Rate Verify\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Refused\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Restarted\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\See Alternative\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\See ED Trauma Flowsheet\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\See OR/Proc Flowsheet\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Stopped\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Waste\" -"\PCORI_MOD\RX_BASIS\PR\02\","\Medication\MAR Result\Wasted\" +\PCORI_MOD\RX_BASIS\PR\OT\,\Medication\MAR Result\ \PCORI_MOD\PDX\P\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\discharge_primary\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\DI\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\AD\,\Diagnosis\\modifier_dx\billing_diagnosis\admit\ From cdde3b9880d64da9698ee758c7c5497feaf9aaf6 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 30 Nov 2016 15:37:05 -0600 Subject: [PATCH 172/507] Failing test showing zero BMI records in CDM (#4335) --- Oracle/cdm_transform_tests.sql | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index d832582..02127f6 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -56,11 +56,16 @@ wt as ( select count(*) qty from pcornet_cdm.vital where wt is not null ), +bmi as ( + select count(*) qty from pcornet_cdm.vital + where original_bmi is not null + ), results as ( select round((ht.qty/total_vital.qty) * 100) pct_ht, - round((wt.qty/total_vital.qty) * 100) pct_wt - from total_vital cross join ht cross join wt + round((wt.qty/total_vital.qty) * 100) pct_wt, + round((bmi.qty/total_vital.qty) * 100) pct_bmi + from total_vital cross join ht cross join wt cross join bmi ) select 'some_height_measurements_4335' query_name , 'Make sure we have at least some height records' description @@ -77,6 +82,14 @@ select 'some_weight_measurements_4335' query_name , wt.qty record_n , results.pct_wt record_pct from total_vital cross join wt cross join results +union all +select 'some_bmi_measurements_4335' query_name + , 'Make sure we have at least some bmi records' description + , case when bmi.qty > 0 then 1 else 0 end pass + , rownum obs + , bmi.qty record_n + , results.pct_bmi record_pct +from total_vital cross join bmi cross join results ; commit; From 4dddece6e88aa0a1f87c42dde364ac9aeb8a87d3 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Wed, 30 Nov 2016 15:53:15 -0600 Subject: [PATCH 173/507] Populate BMI based on visit details in HERON (#4335) --- Oracle/pcornet_mapping.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index b597f7a..358e3a5 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -97,7 +97,7 @@ pcori_path,local_path \PCORI\VITAL\HT\,\i2b2\Visit Details\Vitals\HEIGHT\ \PCORI\VITAL\BP\DIASTOLIC\,\i2b2\Flowsheet\KU\GEN TMP CARD ECHO VITALS T:900426\KU GEN CARD GRP CV COMMON VITALS G:910000 I#2\BLOOD PRESSURE M:5 I#2\DIASTOLIC\ \PCORI\VITAL\BP\SYSTOLIC\,\i2b2\Flowsheet\KU\GEN TMP CARD ECHO VITALS T:900426\KU GEN CARD GRP CV COMMON VITALS G:910000 I#2\BLOOD PRESSURE M:5 I#2\SYSTOLIC\ -\PCORI\VITAL\ORIGINAL_BMI\,\i2b2\Flowsheet\MODEL\VITAL SIGNS COMPLEX T:31010\KU IP GRP ADULT HEIGHT/WEIGHT G:300220 I#4\KU IP ROW BMI M:301070 I#6\ +\PCORI\VITAL\ORIGINAL_BMI\,\i2b2\Visit Details\Vitals\BMI\ \PCORI\VITAL\WT\,\i2b2\Visit Details\Vitals\WEIGHT\ \PCORI\VITAL\TOBACCO\SMOKING\YES\01\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Current Every Day Smoker\ \PCORI\VITAL\TOBACCO\SMOKING\YES\02\,\i2b2\History\Social History\Tobacco Usage\Smoking Tobacco Use\Current Some Day Smoker\ From b802ebd0ca5179b76bd2fef1aa49e5fda6c1ae84 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 30 Nov 2016 16:02:40 -0600 Subject: [PATCH 174/507] Ignore NDCs greater than 11 digits --- Oracle/pcornet_mapping.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 386806e..ecea315 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -420,6 +420,7 @@ select from BLUEHERONMETADATA.HERON_TERMS where c_fullname like '\i2b2\Medications%' and c_basecode like 'NDC:%' + and length(substr(c_basecode, 5)) < 12 -- Make sure we have 11 digit NDC ; delete From e7ce1320459b753c2b8525fc07c22ed81c3ec28c Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 1 Dec 2016 11:07:11 -0600 Subject: [PATCH 175/507] Replaced hardcoded schema / table references with appropriate substitution vars --- Oracle/pcornet_mapping.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index ecea315..9b2c9fd 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -417,7 +417,7 @@ select sourcesystem_cd, valuetype_cd, m_exclusion_cd, c_path, c_symbol, null pcori_basecode, null pcori_cui, -- This might not work substr(c_basecode, 5) pcori_ndc -from BLUEHERONMETADATA.HERON_TERMS +from "&&i2b2_meta_schema"."&&terms_table" where c_fullname like '\i2b2\Medications%' and c_basecode like 'NDC:%' and length(substr(c_basecode, 5)) < 12 -- Make sure we have 11 digit NDC From 2707339b8ff478859a6f2ddf97c21a74ab19f91b Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 1 Dec 2016 11:07:46 -0600 Subject: [PATCH 176/507] Removed unhelpful comment --- Oracle/pcornet_mapping.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 9b2c9fd..a8c6104 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -415,7 +415,7 @@ select c_columnname, c_columndatatype, c_operator, c_dimcode, c_comment, c_tooltip, m_applied_path, update_date, download_date, import_date, sourcesystem_cd, valuetype_cd, m_exclusion_cd, c_path, c_symbol, - null pcori_basecode, null pcori_cui, -- This might not work + null pcori_basecode, null pcori_cui, substr(c_basecode, 5) pcori_ndc from "&&i2b2_meta_schema"."&&terms_table" where c_fullname like '\i2b2\Medications%' From 0e2085eb745b9e676752842d400d9e934652ba37 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Fri, 2 Dec 2016 08:54:30 -0600 Subject: [PATCH 177/507] Backup CDM tables to a different schema (#4407) --- Oracle/backup_cdm.py | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/Oracle/backup_cdm.py b/Oracle/backup_cdm.py index bf5593d..6e31a25 100644 --- a/Oracle/backup_cdm.py +++ b/Oracle/backup_cdm.py @@ -12,29 +12,19 @@ def main(get_cursor): - cursor = get_cursor() + cursor, backup_schema = get_cursor() for table in CDM3_TABLES: if has_rows(cursor, table): - drop_triggers(cursor, table) - drop(cursor, table + '_BAK') - rename(cursor, table, table + '_BAK') + drop(cursor, table, backup_schema) + copy(cursor, table, table, backup_schema) -def drop_triggers(cursor, table): - cursor.execute("""select trigger_name - from user_triggers - where table_name = '%(table)s'""" % - dict(table=table)) - - for res in cursor.fetchall(): - cursor.execute('drop trigger %(trigger)s' % dict(trigger=res[0])) - - -def drop(cursor, table): +def drop(cursor, table, backup_schema): try: - cursor.execute('drop table %(table)s' % dict(table=table)) - except DatabaseError, e: + cursor.execute('drop table %(backup_schema)s.%(table)s' % + dict(backup_schema=backup_schema, table=table)) + except DatabaseError, e: # noqa # Table might not exist pass @@ -51,14 +41,15 @@ def has_rows(cursor, table): return ret -def rename(cursor, table, new_name): - cursor.execute('alter table %(table)s rename to %(new_name)s' % - dict(table=table, new_name=new_name)) +def copy(cursor, table, new_name, backup_schema): + cursor.execute('create table %(backup_schema)s.%(table)s as ' + 'select * from %(table)s' % + dict(backup_schema=backup_schema, table=table)) class MockCursor(object): def __init__(self): - self.fetch_count = 0 + self.fetch_count = 1 def execute(self, sql): print 'execute: ' + sql @@ -75,17 +66,17 @@ def _tcb(): from sys import argv def get_cursor(): - host, port, sid = argv[1:4] + host, port, sid, backup_schema = argv[1:5] user = environ['pcornet_cdm_user'] password = environ['pcornet_cdm'] if '--dry-run' in argv: - return MockCursor() + return MockCursor(), backup_schema else: import cx_Oracle as cx conn = cx.connect(user, password, dsn=cx.makedsn(host, port, sid)) - return conn.cursor() + return conn.cursor(), backup_schema main(get_cursor) _tcb() From 4d15548626bb234cb88439f1b7aa7729b25fe03a Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Fri, 2 Dec 2016 12:49:46 -0600 Subject: [PATCH 178/507] Fix to get DISPENSING facts into Cycle 2 --- Oracle/PCORNetLoader_ora.sql | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index d9e4332..ac8b88f 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1713,7 +1713,25 @@ insert into dispensing ( ,DISPENSE_SUP ---- modifier nval_num ,DISPENSE_AMT -- modifier nval_num -- ,RAW_NDC -) +) +/* Below is the Cycle 2 fix for populating the DISPENSING table */ +select distinct + ibf.patient_num patid, + null prescribingid, + ibf.start_date dispense_date, + substr(ibf.concept_cd, 5) ndc, + null dispense_sup, + null dispense_amt +from i2b2fact ibf +join BLUEHERONMETADATA.pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode +where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' + and length(substr(ibf.concept_cd, 5)) < 12 +; + +/* NOTE: Once DISPENSING related encounters have made it into the CDM (via the visit + dimension) then the original SCILHS code below should work. + select m.patient_num, null,m.start_date, NVL(mo.pcori_ndc,'NA') ,max(supply.nval_num) sup, max(amount.nval_num) amt from i2b2fact m inner join pcornet_med mo @@ -1739,7 +1757,8 @@ inner join encounter enc on enc.encounterid = m.encounter_Num and m.concept_cd = amount.concept_Cd group by m.encounter_num ,m.patient_num, m.start_date, mo.pcori_ndc; - +*/ + execute immediate 'create index dispensing_patid on dispensing (PATID)'; end PCORNetDispensing; From a359a578080455f4aca0ed8010365e96841c9895 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Fri, 2 Dec 2016 13:00:37 -0600 Subject: [PATCH 179/507] Enabled PCORNetDispensing procedure call --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index ac8b88f..14fbe3d 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1866,7 +1866,7 @@ http://listserv.kumc.edu/pipermail/gpc-dev/attachments/20160223/8d79fa70/attachm > LV: the dispensing side [?] is not mandatory? we just did Rx, since that > what we have in our i2b2 */ ---PCORNetDispensing; +PCORNetDispensing; PCORNetHarvest; end pcornetloader; From e0f594ffb9abecc05136994dc9e53669c970168c Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Fri, 2 Dec 2016 13:48:46 -0600 Subject: [PATCH 180/507] Added two, low-bar, tests for DISPENSING ETL --- Oracle/cdm_transform_tests.sql | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index d832582..42cbc8c 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -584,6 +584,38 @@ order by 2 */ +/* "Low bar" test to verify that DISPENSING is not empty */ +insert into test_cases (query_name, description, obs, by_value1, by_value2, record_pct, pass) +select + 'DISPENSING' , + 'Make sure that there are at least 10 dispensing fact' description, + count(*) obs, + null by_value1, + null by_value2, + null record_pct, + case when count(*) >= 10 then 1 else 0 end pass +from dispensing +; + + +/* Test that there are at least a handful of distinct patients associated with + * DISPENSING facts. + */ +insert into test_cases (query_name, description, obs, by_value1, by_value2, record_pct, pass) +select + 'DISPENSING_PATS', + 'Make sure that there are at least 3 distinct patients with dispensing facts', + count(distinct dis.patid) obs, + null by_value1, + null by_value2, + null record_pct, + case when count(distinct dis.patid) >= 3 then 1 else 0 end pass +from dispensing dis +inner join demographic dem + on dis.patid=dem.patid +; + + /* Verify that LAB_RESULT_CM contains data associated with a least half of the labs defined in PMN_LABLOCAL. */ From b1fc110f8892a09953e022f10c4981492955a9b2 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Fri, 2 Dec 2016 14:15:43 -0600 Subject: [PATCH 181/507] Print status/diagnostics while backing up CDM tables (#4407) - As per Dan's suggestion --- Oracle/backup_cdm.py | 48 ++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/Oracle/backup_cdm.py b/Oracle/backup_cdm.py index 6e31a25..de8ef87 100644 --- a/Oracle/backup_cdm.py +++ b/Oracle/backup_cdm.py @@ -15,36 +15,50 @@ def main(get_cursor): cursor, backup_schema = get_cursor() for table in CDM3_TABLES: - if has_rows(cursor, table): - drop(cursor, table, backup_schema) - copy(cursor, table, table, backup_schema) - - -def drop(cursor, table, backup_schema): + bak_schema_table = ('%(backup_schema)s.%(table)s' % + dict(backup_schema=backup_schema, + table=table)) + rcount = count_rows(cursor, table) + print ('%(table)s (to be backed up) currently has %(rcount)s rows.' + % dict(table=table, rcount=rcount)) + + if rcount: + drop(cursor, bak_schema_table) + copy(cursor, table, bak_schema_table) + print ('%(bak_schema_table)s now has %(rows)s rows.' % + dict(bak_schema_table=bak_schema_table, + rows=count_rows(cursor, bak_schema_table))) + + +def drop(cursor, schema_table): + print ('Dropping %(schema_table)s (%(rows)s rows).' % + dict(schema_table=schema_table, + rows=count_rows(cursor, schema_table))) try: - cursor.execute('drop table %(backup_schema)s.%(table)s' % - dict(backup_schema=backup_schema, table=table)) + cursor.execute('drop table %(schema_table)s' % + dict(schema_table=schema_table)) except DatabaseError, e: # noqa # Table might not exist pass -def has_rows(cursor, table): - ret = False +def count_rows(cursor, schema_table): + ret = None try: - cursor.execute('select count(*) from %(table)s' % - dict(table=table)) - if int(cursor.fetchall()[0][0]) > 0: - ret = True + cursor.execute('select count(*) from %(schema_table)s' % + dict(schema_table=schema_table)) + ret = int(cursor.fetchall()[0][0]) except: pass return ret -def copy(cursor, table, new_name, backup_schema): - cursor.execute('create table %(backup_schema)s.%(table)s as ' +def copy(cursor, table, bak_schema_table): + print ('Copying %(table)s to %(bak_schema_table)s.' % + dict(table=table, bak_schema_table=bak_schema_table)) + cursor.execute('create table %(bak_schema_table)s as ' 'select * from %(table)s' % - dict(backup_schema=backup_schema, table=table)) + dict(bak_schema_table=bak_schema_table, table=table)) class MockCursor(object): From d19e8349c4170382909e263daafea2118ff55de9 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Fri, 2 Dec 2016 15:02:42 -0600 Subject: [PATCH 182/507] Moved dispensing tests towards the top of the file as suggested in review --- Oracle/cdm_transform_tests.sql | 65 +++++++++++++++++----------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql index 296a7ab..337f9d5 100644 --- a/Oracle/cdm_transform_tests.sql +++ b/Oracle/cdm_transform_tests.sql @@ -42,6 +42,39 @@ select 'rejected_condition_rows' query_name from total_condition tc cross join err ; + +/* "Low bar" test to verify that DISPENSING is not empty */ +insert into test_cases (query_name, description, obs, by_value1, by_value2, record_pct, pass) +select + 'DISPENSING' , + 'Make sure that there are at least 10 dispensing fact' description, + count(*) obs, + null by_value1, + null by_value2, + null record_pct, + case when count(*) >= 10 then 1 else 0 end pass +from dispensing +; + + +/* Test that there are at least a handful of distinct patients associated with + * DISPENSING facts. + */ +insert into test_cases (query_name, description, obs, by_value1, by_value2, record_pct, pass) +select + 'DISPENSING_PATS', + 'Make sure that there are at least 3 distinct patients with dispensing facts', + count(distinct dis.patid) obs, + null by_value1, + null by_value2, + null record_pct, + case when count(distinct dis.patid) >= 3 then 1 else 0 end pass +from dispensing dis +inner join demographic dem + on dis.patid=dem.patid +; + + /* Test that we have some height/weight measurements */ insert into test_cases (query_name, description, pass, obs, record_n, record_pct) @@ -597,38 +630,6 @@ order by 2 */ -/* "Low bar" test to verify that DISPENSING is not empty */ -insert into test_cases (query_name, description, obs, by_value1, by_value2, record_pct, pass) -select - 'DISPENSING' , - 'Make sure that there are at least 10 dispensing fact' description, - count(*) obs, - null by_value1, - null by_value2, - null record_pct, - case when count(*) >= 10 then 1 else 0 end pass -from dispensing -; - - -/* Test that there are at least a handful of distinct patients associated with - * DISPENSING facts. - */ -insert into test_cases (query_name, description, obs, by_value1, by_value2, record_pct, pass) -select - 'DISPENSING_PATS', - 'Make sure that there are at least 3 distinct patients with dispensing facts', - count(distinct dis.patid) obs, - null by_value1, - null by_value2, - null record_pct, - case when count(distinct dis.patid) >= 3 then 1 else 0 end pass -from dispensing dis -inner join demographic dem - on dis.patid=dem.patid -; - - /* Verify that LAB_RESULT_CM contains data associated with a least half of the labs defined in PMN_LABLOCAL. */ From b01417d3913169c796d4ff462e822455f1e7a3f5 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 5 Dec 2016 13:16:11 -0600 Subject: [PATCH 183/507] Revert "Add ICD10 diagnoses ontology to `pcornet_cdm` #4285" This reverts commit 72f57634481a0147ce775a9ca461a9f0332e3e99. --- Oracle/pcornet_mapping.sql | 42 -------------------------------------- 1 file changed, 42 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 5895da9..df73fb5 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -137,48 +137,6 @@ order by c_hlevel ; -/* Replace PCORNet ICD10 diagnoses hierarchy with the local hierarchy filling in -the pcornet_basecode with the expected values. -*/ - -create or replace view icd10_diag_local_to_pcori as -select '\PCORI\DIAGNOSIS\10\' as pcori_path - , '\i2b2\Diagnoses\ICD10\' as local_path - , 'ICD10:' as pcori_scheme - , 'KUH|DX_ID:' as local_scheme -from dual; - -insert into "&&i2b2_meta_schema".PCORNET_DIAG -with terms_dxi as ( - select - cicd.code dxicd, ht.*, m.* - from "&&i2b2_meta_schema"."&&terms_table" ht - join icd10_diag_local_to_pcori m - on ht.c_fullname like m.local_path||'%' - -- TODO: Stop cheating by going back to Clarity - left join clarity.edg_current_icd10@id cicd - on to_char(cicd.dx_id) = replace(ht.c_basecode, m.local_scheme, '') - order by ht.c_hlevel - ) - -select - td.c_hlevel, - replace(td.c_fullname, local_path, pcori_path) c_fullname, - td.c_name, td.c_synonym_cd, td.c_visualattributes, - td.c_totalnum, td.c_basecode, td.c_metadataxml, td.c_facttablecolumn, td.c_tablename, - td.c_columnname, td.c_columndatatype, td.c_operator, td.c_dimcode, td.c_comment, - td.c_tooltip, td.m_applied_path, td.update_date, td.download_date, td.import_date, - td.sourcesystem_cd, td.valuetype_cd, td.m_exclusion_cd, td.c_path, td.c_symbol, - case - when td.dxicd is not null then td.dxicd - when td.c_basecode like pcori_scheme||'%' then replace(td.c_basecode, pcori_scheme, '') - else null - end pcori_basecode -from terms_dxi td -order by c_hlevel -; - - -- Other diagnosis mappings such as modifiers for PDX, DX_SOURCE. insert into "&&i2b2_meta_schema".PCORNET_DIAG SELECT pcornet_diag.C_HLEVEL+1, From 0dc2dfe00bc3179c96dae2f1e8407a6296efebc8 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 5 Dec 2016 13:17:05 -0600 Subject: [PATCH 184/507] map HERON ICD10 to pcornet as well (#4285) Integrate with existing ICD9 mapping code rather than repeating it. --- Oracle/pcornet_mapping.sql | 59 ++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index df73fb5..b148872 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -98,42 +98,63 @@ update "&&i2b2_meta_schema".pcornet_diag pd set pd.pcori_basecode = 'PDX:P' wher update "&&i2b2_meta_schema".pcornet_diag pd set pd.pcori_basecode = 'PDX:X' where c_fullname = '\PCORI_MOD\PDX\X\'; -/* Replace PCORNet ICD9 diagnoses hierarchy with the local hierarchy filling in +/* Replace PCORNet ICD9, ICD10 diagnoses hierarchy with the local hierarchy filling in the pcornet_basecode with the expected values. */ ---select * +-- select * delete from "&&i2b2_meta_schema".PCORNET_DIAG where c_fullname like '\PCORI\DIAGNOSIS\09\%' + or c_fullname like '\PCORI\DIAGNOSIS\10\%' ; insert into "&&i2b2_meta_schema".PCORNET_DIAG -with terms_dxi as ( - select - cicd.code dxicd, ht.* - from - "&&i2b2_meta_schema"."&&terms_table" ht - -- TODO: Stop cheating by going back to Clarity - left join clarity.edg_current_icd9@id cicd on to_char(cicd.dx_id) = replace(ht.c_basecode, 'KUH|DX_ID:', '') - where c_fullname like '\i2b2\Diagnoses\ICD9\%' order by c_hlevel - ) +with edg9 as ( + select dx_id, code from clarity.edg_current_icd9@id +) +, edg10 as ( + select dx_id, code from clarity.edg_current_icd10@id +) +, heron_dx as ( + select case + when ht.c_basecode like 'KUH|DX_ID:%' + then to_number(replace(ht.c_basecode, 'KUH|DX_ID:', '')) + end dx_id + , ht.* + from "&&i2b2_meta_schema"."&&terms_table" ht + where c_fullname like '\i2b2\Diagnoses\ICD%' +) +, terms_dxi as ( + select case + when ht.c_basecode like 'ICD9:%' then replace(ht.c_basecode, 'ICD9:', '') + when ht.c_basecode like 'ICD10:%' then replace(ht.c_basecode, 'ICD10:', '') + when ht.c_basecode like 'KUH|DX_ID:%' + and ht.c_fullname like '\i2b2\Diagnoses\ICD9\%' + then edg9.code + when ht.c_basecode like 'KUH|DX_ID:%' + and ht.c_fullname like '\i2b2\Diagnoses\ICD10\%' + then edg10.code + end pcori_basecode + , ht.* + from heron_dx ht + left join edg9 + on to_char(edg9.dx_id) = ht.dx_id + left join edg10 + on to_char(edg10.dx_id) = ht.dx_id +) select - td.c_hlevel, - replace(td.c_fullname, '\i2b2\Diagnoses\ICD9\', '\PCORI\DIAGNOSIS\09\') c_fullname, + td.c_hlevel, + replace(replace(td.c_fullname, '\i2b2\Diagnoses\ICD9\', '\PCORI\DIAGNOSIS\09\') + , '\i2b2\Diagnoses\ICD10\', '\PCORI\DIAGNOSIS\10\') c_fullname, td.c_name, td.c_synonym_cd, td.c_visualattributes, td.c_totalnum, td.c_basecode, td.c_metadataxml, td.c_facttablecolumn, td.c_tablename, td.c_columnname, td.c_columndatatype, td.c_operator, td.c_dimcode, td.c_comment, td.c_tooltip, td.m_applied_path, td.update_date, td.download_date, td.import_date, td.sourcesystem_cd, td.valuetype_cd, td.m_exclusion_cd, td.c_path, td.c_symbol, - case - when td.dxicd is not null then td.dxicd - when td.c_basecode like 'ICD9:%' then replace(td.c_basecode, 'ICD9:', '') - else null - end pcori_basecode + td.pcori_basecode from terms_dxi td -order by c_hlevel ; From 498d6a31645a59d5c3fada6d2049279dd9bda9d4 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 5 Dec 2016 14:35:53 -0600 Subject: [PATCH 185/507] normalize newlines back to CRLF --- Oracle/PCORNetLoader_ora.sql | 44 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 14fbe3d..d6e389a 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1713,25 +1713,25 @@ insert into dispensing ( ,DISPENSE_SUP ---- modifier nval_num ,DISPENSE_AMT -- modifier nval_num -- ,RAW_NDC -) -/* Below is the Cycle 2 fix for populating the DISPENSING table */ -select distinct - ibf.patient_num patid, - null prescribingid, - ibf.start_date dispense_date, - substr(ibf.concept_cd, 5) ndc, - null dispense_sup, - null dispense_amt -from i2b2fact ibf -join BLUEHERONMETADATA.pcornet_med pnm - on ibf.modifier_cd=pnm.c_basecode -where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' - and length(substr(ibf.concept_cd, 5)) < 12 -; - -/* NOTE: Once DISPENSING related encounters have made it into the CDM (via the visit - dimension) then the original SCILHS code below should work. - +) +/* Below is the Cycle 2 fix for populating the DISPENSING table */ +select distinct + ibf.patient_num patid, + null prescribingid, + ibf.start_date dispense_date, + substr(ibf.concept_cd, 5) ndc, + null dispense_sup, + null dispense_amt +from i2b2fact ibf +join BLUEHERONMETADATA.pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode +where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' + and length(substr(ibf.concept_cd, 5)) < 12 +; + +/* NOTE: Once DISPENSING related encounters have made it into the CDM (via the visit + dimension) then the original SCILHS code below should work. + select m.patient_num, null,m.start_date, NVL(mo.pcori_ndc,'NA') ,max(supply.nval_num) sup, max(amount.nval_num) amt from i2b2fact m inner join pcornet_med mo @@ -1757,8 +1757,8 @@ inner join encounter enc on enc.encounterid = m.encounter_Num and m.concept_cd = amount.concept_Cd group by m.encounter_num ,m.patient_num, m.start_date, mo.pcori_ndc; -*/ - +*/ + execute immediate 'create index dispensing_patid on dispensing (PATID)'; end PCORNetDispensing; @@ -1866,7 +1866,7 @@ http://listserv.kumc.edu/pipermail/gpc-dev/attachments/20160223/8d79fa70/attachm > LV: the dispensing side [?] is not mandatory? we just did Rx, since that > what we have in our i2b2 */ -PCORNetDispensing; +PCORNetDispensing; PCORNetHarvest; end pcornetloader; From 1181945734d59856157161b992d408dd8f27db6e Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 5 Dec 2016 14:39:52 -0600 Subject: [PATCH 186/507] un-quote `insert into diagnosis ...` I think SCILHS has gone away from this quoting style and I certainly don't want to deal with it when working on ICD10 stuff. --- Oracle/PCORNetLoader_ora.sql | 49 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index d6e389a..6be6d92 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1008,32 +1008,31 @@ PMN_EXECUATESQL(sqltext); execute immediate 'create index pdxfact_idx on pdxfact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('PDXFACT'); -sqltext := 'insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, pdx) '|| -'select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, factline.start_date, factline.provider_id, diag.pcori_basecode, '|| -'SUBSTR(diag.c_fullname,18,2) dxtype, '|| -' CASE WHEN enc_type=''AV'' THEN ''FI'' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,'':'')+1,2) ,''NI'') END, '|| -' CASE WHEN enc_type in (''IP'', ''IS'') -- PDX is "relevant only on IP and IS encounters" - THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, '':'')+1,2),''NI'') - ELSE ''X'' END '|| -'from i2b2fact factline '|| -'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| -' left outer join sourcefact '|| -'on factline.patient_num=sourcefact.patient_num '|| -'and factline.encounter_num=sourcefact.encounter_num '|| -'and factline.provider_id=sourcefact.provider_id '|| -'and factline.concept_cd=sourcefact.concept_Cd '|| -'and factline.start_date=sourcefact.start_Date '|| -'left outer join pdxfact '|| -'on factline.patient_num=pdxfact.patient_num '|| -'and factline.encounter_num=pdxfact.encounter_num '|| -'and factline.provider_id=pdxfact.provider_id '|| -'and factline.concept_cd=pdxfact.concept_cd '|| -'and factline.start_date=pdxfact.start_Date '|| -'inner join pcornet_diag diag on diag.c_basecode = factline.concept_cd '|| -'where diag.c_fullname like ''\PCORI\DIAGNOSIS\%'' '|| -'and (sourcefact.c_fullname like ''\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\%'' or sourcefact.c_fullname is null) '; +insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, pdx) +select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, factline.start_date, factline.provider_id, diag.pcori_basecode, +SUBSTR(diag.c_fullname,18,2) dxtype, + CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END, + CASE WHEN enc_type in ('IP', 'IS') -- PDX is "relevant only on IP and IS encounters" + THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, ':')+1,2),'NI') + ELSE 'X' END +from i2b2fact factline +inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + left outer join sourcefact +on factline.patient_num=sourcefact.patient_num +and factline.encounter_num=sourcefact.encounter_num +and factline.provider_id=sourcefact.provider_id +and factline.concept_cd=sourcefact.concept_Cd +and factline.start_date=sourcefact.start_Date +left outer join pdxfact +on factline.patient_num=pdxfact.patient_num +and factline.encounter_num=pdxfact.encounter_num +and factline.provider_id=pdxfact.provider_id +and factline.concept_cd=pdxfact.concept_cd +and factline.start_date=pdxfact.start_Date +inner join pcornet_diag diag on diag.c_basecode = factline.concept_cd +where diag.c_fullname like '\PCORI\DIAGNOSIS\%' +and (sourcefact.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\%' or sourcefact.c_fullname is null); -PMN_EXECUATESQL(sqltext); execute immediate 'create index diagnosis_patid on diagnosis (PATID)'; execute immediate 'create index diagnosis_encounterid on diagnosis (ENCOUNTERID)'; From fbf416f944aca0d8d162ba9eb94042be9634baa4 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 5 Dec 2016 15:30:50 -0600 Subject: [PATCH 187/507] get encounter.admit_date, providerid to diagnosis note upstream fix in 1254d76d62d4d68dca37849ff55552fa1e0b7e98 --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 6be6d92..03716a7 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1009,7 +1009,7 @@ execute immediate 'create index pdxfact_idx on pdxfact (patient_num, encounter_n GATHER_TABLE_STATS('PDXFACT'); insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, pdx) -select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, factline.start_date, factline.provider_id, diag.pcori_basecode, +select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, enc.admit_date, factline.provider_id, diag.pcori_basecode, SUBSTR(diag.c_fullname,18,2) dxtype, CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END, CASE WHEN enc_type in ('IP', 'IS') -- PDX is "relevant only on IP and IS encounters" From 67f472b53f19b430e554318e6d6a4790a7ca9851 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 5 Dec 2016 15:38:56 -0600 Subject: [PATCH 188/507] map to ICD9 vs. ICD10 based on admit_date (#4285) --- Oracle/PCORNetLoader_ora.sql | 52 +++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 03716a7..50c5630 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1009,12 +1009,48 @@ execute immediate 'create index pdxfact_idx on pdxfact (patient_num, encounter_n GATHER_TABLE_STATS('PDXFACT'); insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, pdx) -select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, enc.admit_date, factline.provider_id, diag.pcori_basecode, -SUBSTR(diag.c_fullname,18,2) dxtype, - CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END, +/* KUMC started billing with ICD10 on Oct 1, 2015. */ +with icd10_transition as ( + select date '2015-10-01' as cutoff from dual +) +/* DX_IDs may have mappings to both ICD9 and ICD10 */ +, has9 as ( + select distinct c_basecode, pcori_basecode icd9_code + from blueheronmetadata.pcornet_diag diag + where diag.c_fullname like '\PCORI\DIAGNOSIS\09%' +) +, has10 as ( + select distinct c_basecode, pcori_basecode icd10_code + from blueheronmetadata.pcornet_diag diag + where diag.c_fullname like '\PCORI\DIAGNOSIS\10%' +) +, has9_and_10 as ( + select has9.c_basecode, has9.icd9_code, has10.icd10_code + from has9 join has10 on has9.c_basecode = has10.c_basecode +) +, diag as ( -- replace diag rows by 9_and_10 rows and avoid dups + select distinct diag.c_basecode, diag.pcori_basecode, icd9_code, icd10_code + , case when has9_and_10.c_basecode is null then SUBSTR(diag.c_fullname,18,2) else null end dx_type + from pcornet_diag diag + left join has9_and_10 on has9_and_10.c_basecode = diag.c_basecode + where diag.c_fullname like '\PCORI\DIAGNOSIS\%' +) + +select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, enc.admit_date, factline.provider_id + , case + when diag.pcori_basecode is not null then diag.pcori_basecode + when enc.admit_date >= icd10_transition.cutoff then diag.icd10_code + else diag.icd9_code + end dx + , case + when diag.dx_type is not null then diag.dx_type + when enc.admit_date >= icd10_transition.cutoff then '10' + else '09' + end dxtype, + CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END dx_source, CASE WHEN enc_type in ('IP', 'IS') -- PDX is "relevant only on IP and IS encounters" THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, ':')+1,2),'NI') - ELSE 'X' END + ELSE 'X' END PDX from i2b2fact factline inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num left outer join sourcefact @@ -1029,9 +1065,11 @@ and factline.encounter_num=pdxfact.encounter_num and factline.provider_id=pdxfact.provider_id and factline.concept_cd=pdxfact.concept_cd and factline.start_date=pdxfact.start_Date -inner join pcornet_diag diag on diag.c_basecode = factline.concept_cd -where diag.c_fullname like '\PCORI\DIAGNOSIS\%' -and (sourcefact.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\%' or sourcefact.c_fullname is null); +inner join diag on diag.c_basecode = factline.concept_cd +cross join icd10_transition +where (sourcefact.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\%' or sourcefact.c_fullname is null) +-- order by enc.admit_date desc +; execute immediate 'create index diagnosis_patid on diagnosis (PATID)'; From 50806ca2aef2e7540c4c5fd495374aae7a76424d Mon Sep 17 00:00:00 2001 From: PCORnet Coordinating Center Date: Mon, 5 Dec 2016 16:11:27 -0600 Subject: [PATCH 189/507] Copy of parseable CDM v3 specification --- ...mon-Data-Model-v3dot0-parseable-fields.csv | 954 ++++++++++++++++++ 1 file changed, 954 insertions(+) create mode 100644 2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv diff --git a/2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv b/2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv new file mode 100644 index 0000000..bf80f40 --- /dev/null +++ b/2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv @@ -0,0 +1,954 @@ +TABLE_NAME,FIELD_NAME,RDBMS_DATA_TYPE,SAS_DATA_TYPE,DATA_FORMAT,REPLICATED,UNIT_OF_MEASURE,VALUE_SET,VALUE_DESCRIPTION,DEFINITION,CDM_ORDER +DEMOGRAPHIC,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary person-level identifier used to link across tables. + +PATID is a pseudoidentifier with a consistent crosswalk to the true identifier retained by the source Data Partner. For analytical data sets requiring patient-level data, only the pseudoidentifier is used to link across all information belonging to a patient. + +The PATID must be unique within the data source being queried. Creating a unique identifier within a CDRN would be beneficial and acceptable. The PATID is not the basis for linkages across partners.",1 +DEMOGRAPHIC,BIRTH_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date of birth.,2 +DEMOGRAPHIC,BIRTH_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Time of birth.,3 +DEMOGRAPHIC,SEX,RDBMS Text(2),SAS Char(2),,NO,,A;F;M;NI;UN;OT,"A = Ambiguous +F = Female +M = Male +NI = No information +UN = Unknown +OT = Other","Administrative sex. + +v2.0 guidance added: The ?Ambiguous? category may be used for individuals who are physically undifferentiated from birth. The ?Other? category may be used for individuals who are undergoing gender re-assignment.",4 +DEMOGRAPHIC,HISPANIC,RDBMS Text(2),SAS Char(2),,NO,,Y;N;R;NI;UN;OT,"Y = Yes +N = No +R = Refuse to answer +NI = No information +UN = Unknown +OT = Other","A person of Cuban, Mexican, Puerto Rican, South or Central American, or other Spanish culture or origin, regardless of race. + +v2.0 amendment: The new categorical value of ?Refuse to answer? has been added.",5 +DEMOGRAPHIC,RACE,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;05;06;07;NI;UN;OT,"01 = American Indian or Alaska Native +02 = Asian +03 = Black or African American +04 = Native Hawaiian or Other Pacific Islander +05 = White +06 = Multiple race +07 = Refuse to answer +NI = No information +UN = Unknown +OT = Other","Please use only one race value per patient. + +Details of categorical definitions: +American Indian or Alaska Native: A person having origins in any of the original peoples of North and South America (including Central America), and who maintains tribal affiliation or community attachment. + +Asian: A person having origins in any of the original peoples of the Far East, Southeast Asia, or the Indian subcontinent including, for example, Cambodia, China, India, Japan, Korea, Malaysia, Pakistan, the Philippine Islands, Thailand, and Vietnam. + +Black or African American: A person having origins in any of the black racial groups of Africa. + +Native Hawaiian or Other Pacific Islander: A person having origins in any of the original peoples of Hawaii, Guam, Samoa, or other Pacific Islands. + +White: A person having origins in any of the original peoples of Europe, the Middle East, or North Africa.",6 +DEMOGRAPHIC,BIOBANK_FLAG,RDBMS Text(1),SAS Char(1),,NO,,Y;N,"Y = Yes +N = No","Flag to indicate that one or more biobanked specimens are stored and available for research use. Examples of biospecimens could include plasma, urine, or tissue. If biospecimens are available, locally maintained ?mapping tables? would be necessary to map between the DEMOGRAPHIC record and the originating biobanking system(s). + +If no known biobanked specimens are available, this field should be marked ?No?.",7 +DEMOGRAPHIC,RAW_SEX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",8 +DEMOGRAPHIC,RAW_HISPANIC,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",9 +DEMOGRAPHIC,RAW_RACE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",10 +ENROLLMENT,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables.,11 +ENROLLMENT,ENR_START_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date of the beginning of the enrollment period. If the exact date is unknown, use the first day of the month. + +For implementation of the CDM, a long span of longitudinal data is desirable; however, especially for historical data more than a decade old, the appropriate beginning date should be determined by the partner?s knowledge of the validity and usability of the data. More specific guidance can be provided through implementation discussions.",12 +ENROLLMENT,ENR_END_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date of the end of the enrollment period. If the exact date is unknown, use the last day of the month.",13 +ENROLLMENT,CHART,RDBMS Text(1),SAS Char(1),,NO,,Y;N,"Y = Yes +N = No","Chart abstraction flag is intended to answer the question, ""Are you able to request (or review) charts for this person?"" This flag does not address chart availability. Mark as ""Yes"" if there are no contractual or other restrictions between you and the individual (or sponsor) that would prohibit you from requesting any chart for this member. + +Note: This field is most relevant for health insurers that can request charts from affiliated providers. This field allows exclusion of patients from studies that require chart review to validate exposures and/or outcomes. It identifies patients for whom charts are never available and for whom the chart can never be requested.",14 +ENROLLMENT,ENR_BASIS,RDBMS Text(1),SAS Char(1),,NO,,I;G;A;E,"I = Insurance +G = Geography +A = Algorithmic +E = Encounter-based","When insurance information is not available but complete capture can be asserted some other way, please identify the basis on which complete capture is defined. Additional information on the approach identified will be required from each partner. + +ENR_BASIS is a property of the time period defined. A patient can have multiple entries in the table. + +Details of categorical definitions: +Insurance: The start and stop dates are based upon the concept of enrollment with health plan insurance + +Geography: An assertion of complete data capture between the start and end dates based upon geographic characteristics, such as regional isolation + +Algorithmic: An assertion of complete data capture between the start and end dates, based on a locally developed or applied algorithm, often using multiple criteria + +Encounter-based: The start and stop dates are populated from the earliest-observed encounter and latest-observed encounter.",15 +ENCOUNTER,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier. Used to link across tables, including the ENCOUNTER, DIAGNOSIS, and PROCEDURE tables.",16 +ENCOUNTER,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables.,17 +ENCOUNTER,ADMIT_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Encounter or admission date.,18 +ENCOUNTER,ADMIT_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Encounter or admission time.,19 +ENCOUNTER,DISCHARGE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Discharge date. Should be populated for all Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),20 +ENCOUNTER,DISCHARGE_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Discharge time.,21 +ENCOUNTER,PROVIDERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Provider code for the provider who is most responsible for this encounter. For encounters with multiple providers choose one so the encounter can be linked to the diagnosis and procedure tables. As with the PATID, the provider code is a pseudoidentifier with a consistent crosswalk to the real identifier.",22 +ENCOUNTER,FACILITY_LOCATION,RDBMS Text(3),SAS Char(3),,NO,,,,Geographic location (3 digit zip code). Should be null if not recorded in source system (modification made from ?blank? to ?null? in v3.0).,23 +ENCOUNTER,ENC_TYPE,RDBMS Text(2),SAS Char(2),,NO,,AV;ED;EI;IP;IS;OA;NI;UN;OT,"AV = Ambulatory Visit +ED = Emergency Department +EI = Emergency Department Admit to Inpatient Hospital Stay (permissible substitution) +IP = Inpatient Hospital Stay +IS = Non-Acute Institutional Stay +OA = Other Ambulatory Visit +NI = No information +UN = Unknown +OT = Other","Encounter type. + +Details of categorical definitions: +Ambulatory Visit: Includes visits at outpatient clinics, physician offices, same day/ambulatory surgery centers, urgent care facilities, and other same-day ambulatory hospital encounters, but excludes emergency department encounters. + +Emergency Department (ED): Includes ED encounters that become inpatient stays (in which case inpatient stays would be a separate encounter). Excludes urgent care visits. ED claims should be pulled before hospitalization claims to ensure that ED with subsequent admission won't be rolled up in the hospital event. + +Emergency Department Admit to Inpatient Hospital Stay: Permissible substitution for preferred state of separate ED and IP records. Only for use with data sources where the individual records for ED and IP cannot be distinguished (new to v2.0). + +Inpatient Hospital Stay: Includes all inpatient stays, including: same-day hospital discharges, hospital transfers, and acute hospital care where the discharge is after the admission date. + +Non-Acute Institutional Stay: Includes hospice, skilled nursing facility (SNF), rehab center, nursing home, residential, overnight non-hospital dialysis and other non-hospital stays. + +Other Ambulatory Visit: Includes other non-overnight AV encounters such as hospice visits, home health visits, skilled nursing facility visits, other non-hospital visits, as well as telemedicine, telephone and email consultations. May also include ""lab only"" visits (when a lab is ordered outside of a patient visit), ""pharmacy only"" (e.g., when a patient has a refill ordered without a face-to-face visit), ""imaging only"", etc.",24 +ENCOUNTER,FACILITYID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary local facility code that identifies the hospital or clinic. Used for chart abstraction and validation. + +FACILITYID can be a true identifier, or a pseudoidentifier with a consistent crosswalk to the true identifier retained by the source Data Partner.",25 +ENCOUNTER,DISCHARGE_DISPOSITION,RDBMS Text(2),SAS Char(2),,NO,,A;E;NI;UN;OT,"A = Discharged alive +E = Expired +NI = No information +UN = Unknown +OT = Other",Vital status at discharge. Should be populated for Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),26 +ENCOUNTER,DISCHARGE_STATUS,RDBMS Text(2),SAS Char(2),,NO,,AF;AL;AM;AW;EX;HH;HO;HS;IP;NH;RH;RS;SH;SN;NI;UN;OT,"AF = Adult Foster Home +AL = Assisted Living Facility +AM = Against Medical Advice +AW = Absent without leave +EX = Expired +HH = Home Health +HO = Home / Self Care +HS = Hospice +IP = Other Acute Inpatient Hospital +NH = Nursing Home (Includes ICF) +RH = Rehabilitation Facility +RS = Residential Facility +SH = Still In Hospital +SN = Skilled Nursing Facility +NI = No information +UN = Unknown +OT = Other",Discharge status. Should be populated for Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),27 +ENCOUNTER,DRG,RDBMS Text(3),SAS Char(3),,NO,,,,"3-digit Diagnosis Related Group (DRG). Should be populated for IP and IS encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for AV or OA encounters. Use leading zeroes for codes less than 100. (Additional guidance added in v3.0 for the EI encounter type.) + +The DRG is used for reimbursement for inpatient encounters. It is a Medicare requirement that combines diagnoses into clinical concepts for billing. Frequently used in observational data analyses. +",28 +ENCOUNTER,DRG_TYPE,RDBMS Text(2),SAS Char(2),,NO,,01;02;NI;UN;OT,"01 = CMS-DRG (old system) +02 = MS-DRG (current system) +NI = No information +UN = Unknown +OT = Other",DRG code version. MS-DRG (current system) began on 10/1/2007. Should be populated for IP and IS encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for AV or OA encounters. (Additional guidance added in v3.0 for the EI encounter type.),29 +ENCOUNTER,ADMITTING_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,AF;AL;AV;ED;HH;HO;HS;IP;NH;RH;RS;SN;NI;UN;OT,"AF = Adult Foster Home +AL = Assisted Living Facility +AV = Ambulatory Visit +ED = Emergency Department +HH = Home Health +HO = Home / Self Care +HS = Hospice +IP = Other Acute Inpatient Hospital +NH = Nursing Home (Includes ICF) +RH = Rehabilitation Facility +RS = Residential Facility +SN = Skilled Nursing Facility +NI = No information +UN = Unknown +OT = Other",Admitting source. Should be populated for Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),30 +ENCOUNTER,RAW_SITEID,RDBMS Text(x),SAS Char(x),,NO,,,,"This field is new to v2.0. + +Optional field for locally-defined identifier intended for local use; for example, where a network may have multiple sites contributing to a central data repository. + +This attribute may be sensitive in certain contexts; the intent is for internal network use only, and not to enable site quality comparisons.",31 +ENCOUNTER,RAW_ENC_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",32 +ENCOUNTER,RAW_DISCHARGE_DISPOSITION,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",33 +ENCOUNTER,RAW_DISCHARGE_STATUS,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",34 +ENCOUNTER,RAW_DRG_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",35 +ENCOUNTER,RAW_ADMITTING_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",36 +DIAGNOSIS,DIAGNOSISID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",37 +DIAGNOSIS,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,38 +DIAGNOSIS,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. Used to link across tables.,39 +DIAGNOSIS,ENC_TYPE,RDBMS Text(2),SAS Char(2),,YES,,AV;ED;EI;IP;IS;OA;NI;UN;OT,"AV = Ambulatory Visit +ED = Emergency Department +EI=Emergency Department Admit to Inpatient Hospital Stay (permissible substitution) +IP = Inpatient Hospital Stay +IS = Non-Acute Institutional Stay +OA = Other Ambulatory Visit +NI = No information +UN = Unknown +OT = Other",Please note: This is a field replicated from the ENCOUNTER table. See the ENCOUNTER table for definitions. ,40 +DIAGNOSIS,ADMIT_DATE,RDBMS Date,SAS Date (Numeric),,YES,DATE,,,Please note: This is a field replicated from the ENCOUNTER table. See the ENCOUNTER table for definitions. ,41 +DIAGNOSIS,PROVIDERID,RDBMS Text(x),SAS Char(x),,YES,,,,Please note: This is a field replicated from the ENCOUNTER table. See the ENCOUNTER table for definitions. ,42 +DIAGNOSIS,DX,RDBMS Text(18),SAS Char(18),,NO,,,,"Diagnosis code. + +Leading zeroes and different levels of decimal precision are permissible in this field. Please populate the exact textual value of this diagnosis code, but remove source-specific suffixes and prefixes. Other codes should be listed as recorded in the source data.",43 +DIAGNOSIS,DX_TYPE,RDBMS Text(2),SAS Char(2),,NO,,09;10;11;SM;NI;UN;OT,"09 = ICD-9-CM +10 = ICD-10-CM +11 = ICD-11-CM +SM = SNOMED CT +NI = No information +UN = Unknown +OT = Other","Diagnosis code type. + +We provide values for ICD and SNOMED code types. Other code types will be added as new terminologies are more widely used. + +Please note: The ?Other? category is meant to identify internal use ontologies and codes.",44 +DIAGNOSIS,DX_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,AD;DI;FI;IN;NI;UN;OT,"AD = Admitting +DI = Discharge +FI = Final +IN = Interim +NI = No information +UN = Unknown +OT = Other","Classification of diagnosis source. We include these categories to allow some flexibility in implementation. The context is to capture available diagnoses recorded during a specific encounter. It is not necessary to populate interim diagnoses unless readily available. + +Ambulatory encounters would generally be expected to have a source of ?Final.?",45 +DIAGNOSIS,PDX,RDBMS Text(2),SAS Char(2),,NO,,P;S;X;NI;UN;OT,"P = Principal +S = Secondary +X = Unable to Classify +NI = No information +UN = Unknown +OT = Other","Principal discharge diagnosis flag. Relevant only on IP and IS encounters. + +For ED, AV, and OA encounter types, mark as X = Unable to Classify. (Billing systems do not require a primary diagnosis for ambulatory visits (eg, professional services)) + +One principal diagnosis per encounter is expected, although in some instances more than one diagnosis may be flagged as principal. (Modification made in v3.0 to specify ?per encounter.?)",46 +DIAGNOSIS,RAW_DX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",47 +DIAGNOSIS,RAW_DX_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",48 +DIAGNOSIS,RAW_DX_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",49 +DIAGNOSIS,RAW_PDX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",50 +PROCEDURES,PROCEDURESID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",51 +PROCEDURES,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,52 +PROCEDURES,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. Used to link across tables.,53 +PROCEDURES,ENC_TYPE,RDBMS Text(2),SAS Char(2),,YES,,AV;ED;EI;IP;IS;OA;NI;UN;OT,"AV = Ambulatory Visit +ED = Emergency Department +EI=Emergency Department Admit to Inpatient Hospital Stay (permissible substitution) +IP = Inpatient Hospital Stay +IS = Non-Acute Institutional Stay +OA = Other Ambulatory Visit +NI = No information +UN = Unknown +OT = Other",Please note: This is a field replicated from the ENCOUNTER table. See ENCOUNTER table for definitions.,54 +PROCEDURES,ADMIT_DATE,RDBMS Date,SAS Date (Numeric),,YES,DATE,,,Please note: This is a field replicated from the ENCOUNTER table. See ENCOUNTER table for definitions.,55 +PROCEDURES,PROVIDERID,RDBMS Text(x),SAS Char(x),,YES,,,,Please note: This is a field replicated from the ENCOUNTER table. See ENCOUNTER table for definitions.,56 +PROCEDURES,PX_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"New to v2.0. + +Date the procedure was performed.",57 +PROCEDURES,PX,RDBMS Text(11),SAS Char(11),,NO,,,,Procedure code.,58 +PROCEDURES,PX_TYPE,RDBMS Text(2),SAS Char(2),,NO,,09;10;11;C2;C3;C4;H3;HC;LC;ND;RE;NI;UN;OT,"09 = ICD-9-CM +10 = ICD-10-PCS +11 = ICD-11-PCS +C2 = CPT Category II +C3 = CPT Category III +C4 = CPT-4 (i.e., HCPCS Level I) +H3 = HCPCS Level III +HC = HCPCS (i.e., HCPCS Level II) +LC = LOINC +ND = NDC +RE = Revenue +NI = No information +UN = Unknown +OT = Other","Procedure code type. + +We include a number of code types for flexibility, but the basic requirement that the code refer to a medical procedure remains. + +Revenue codes are a standard concept in Medicare billing and can be useful for defining care settings. If those codes are available they can be included. + +Medications administered by clinicians can be captured in billing data and Electronic Health Records (EHRs) as HCPCS procedure codes. Administration (infusion) of chemotherapy is an example. + +We are now seeing NDCs captured as part of procedures because payers are demanding it for payment authorization. Inclusion of this code type enables those data partners that capture the NDC along with the procedure to include the data. + +Please note: The ?Other? category is meant to identify internal use ontologies and codes.",59 +PROCEDURES,PX_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,OD;BI;CL;NI;UN;OT,"OD=Order +BI=Billing +CL=Claim +NI=No information +UN=Unknown +OT=Other","New to v2.0. + +Source of the procedure information. + +Order and billing pertain to internal healthcare processes and data sources. Claim pertains to data from the bill fulfillment, generally data sources held by insurers and other health plans.",60 +PROCEDURES,RAW_PX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",61 +PROCEDURES,RAW_PX_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",62 +VITAL,VITALID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique VITAL record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. + +(New field added to v3.0.)",63 +VITAL,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,64 +VITAL,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. This is an optional relationship; the ENCOUNTERID should generally be present if the vitals were measured as part of healthcare delivery captured by this datamart (guidance added in v3.0).,65 +VITAL,MEASURE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date of vitals measure.,66 +VITAL,MEASURE_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Time of vitals measure.,67 +VITAL,VITAL_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,PR;PD;HC;HD;NI;UN;OT,"PR = Patient-reported +PD=Patient device direct feed +HC = Healthcare delivery setting +HD=Healthcare device direct feed +NI = No information +UN = Unknown +OT = Other","Please note: The ?Patient-reported? category can include reporting by patient?s family or guardian. + +v2.0 amendment: The new categorical value of PD and HD have been added. + +v2.0 guidance added with slight modification in v3.0: If unknown whether data are received directly from a device feed, use the more general context (such as patient-reported or healthcare delivery setting).",68 +VITAL,HT,RDBMS Number(8),NUMERIC(8),,NO,INCH,,,"Height (in inches) measured by standing. Only populated if measure was taken on this date. If missing, this value should be null. Decimal precision is permissible. (Modification of wording made from ?blank? to ?null? in v3.0).",69 +VITAL,WT,RDBMS Number(8),NUMERIC(8),,NO,POUND,,,"Weight (in pounds). Only populated if measure was taken on this date. If missing, this value should be null. Decimal precision is permissible. (Modification of wording made from ?blank? to ?null? in v3.0).",70 +VITAL,DIASTOLIC,RDBMS Number(4),NUMERIC(),,NO,MMHG,,,"Diastolic blood pressure (in mmHg). Only populated if measure was taken on this date. If missing, this value should be null. (Modification of wording made from ?blank? to ?null? in v3.0).",71 +VITAL,SYSTOLIC,RDBMS Number(4),NUMERIC(4),,NO,MMHG,,,"Systolic blood pressure (in mmHg). Only populated if measure was taken on this date. If missing, this value should be null. (Modification of wording made from ?blank? to ?null? in v3.0).",72 +VITAL,ORIGINAL_BMI,RDBMS Number(8),NUMERIC(8),,NO,,,,"BMI if calculated in the source system. + +Important: Please do not calculate BMI during CDM implementation. This field should only reflect originating source system calculations, if height and weight are not stored in the source.",73 +VITAL,BP_POSITION,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;NI;UN;OT,"01 = Sitting +02 = Standing +03 = Supine +NI = No information +UN = Unknown +OT = Other",Position for orthostatic blood pressure. This value should be null if blood pressure was not measured. (Modification of wording made from ?blank? to ?null? in v3.0).,74 +VITAL,SMOKING,RDBMS Text(2),SAS Char(2),,,,01;02;03;04;05;06;07;08;NI;UN;OT,"01=Current every day smoker +02=Current some day smoker +03=Former smoker +04=Never smoker +05=Smoker, current status unknown +06=Unknown if ever smoked +07=Heavy tobacco smoker +08=Light tobacco smoker +NI=No information +UN=Unknown +OT=Other","This field is new to v3.0. + +Indicator for any form of tobacco that is smoked. + +Per Meaningful Use guidance, ??smoking status includes any form of tobacco that is smoked, but not all tobacco use.? + +??Light smoker? is interpreted to mean less than 10 cigarettes per day, or an equivalent (but less concretely defined) quantity of cigar or pipe smoke. ?Heavy smoker? is interpreted to mean greater than 10 cigarettes per day or an equivalent (but less concretely defined) quantity of cigar or pipe smoke.? + +??we understand that a ?current every day smoker? or ?current some day smoker? is an individual who has smoked at least 100 cigarettes during his/her lifetime and still regularly smokes every day or periodically, yet consistently; a ?former smoker? would be an individual who has smoked at least 100 cigarettes during his/her lifetime but does not currently smoke; and a ?never smoker? would be an individual who has not smoked 100 or more cigarettes during his/her lifetime.? + +http://www.healthit.gov/sites/default/files/standards-certification/2014-edition-draft-test-procedures/170-314-a-11-smoking-status-2014-test-procedure-draft-v1.0.pdf +[retrieved May 11, 2015] +",75 +VITAL,TOBACCO,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;06;NI;UN;OT,"01=Current user +02=Never +03=Quit/former user +04=Passive or environmental exposure +06=Not asked +NI=No information +UN=Unknown +OT=Other +","This field is new to v2.0 with revised value set and field definition in v3.0. + +Indicator for any form of tobacco.",76 +VITAL,TOBACCO_TYPE,RDBMS Text(2),SAS Char(2),,NO,,"01;02;03;04;05;NI;UN;OT +","01=Smoked tobacco only +02=Non-smoked tobacco only +03=Use of both smoked and non-smoked tobacco products +04=None +05=Use of smoked tobacco but no information about non-smoked tobacco use +NI=No information +UN=Unknown +OT=Other +","This field is new to v2.0, with revised value set in v3.0. + +Type(s) of tobacco used.",77 +VITAL,RAW_DIASTOLIC,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to formatting into the PCORnet CDM.",78 +VITAL,RAW_SYSTOLIC,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to formatting into the PCORnet CDM.",79 +VITAL,RAW_BP_POSITION,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",80 +VITAL,RAW_SMOKING,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set. New to v3.0.",81 +VITAL,RAW_TOBACCO,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set. New to v2.0.",82 +VITAL,RAW_TOBACCO_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set. New to v2.0.",83 +DISPENSING,DISPENSINGID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",84 +DISPENSING,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,85 +DISPENSING,PRESCRIBINGID,RDBMS Text(x),SAS Char(x),,,,,,"This is an optional relationship to the PRESCRIBING table, and may not be generally available. One prescribing order may generate multiple dispensing records. New field added in v3.0.",86 +DISPENSING,DISPENSE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Dispensing date (as close as possible to date the person received the dispensing).,87 +DISPENSING,NDC,RDBMS Text(11),SAS Char(11),,NO,,,,"National Drug Code in the 11-digit, no-dash, HIPAA format. + +Please expunge any place holders (such as dashes or extra digits). + +If needed, guidance on normalization for other forms of NDC can be found: http://www.nlm.nih.gov/research/umls/rxnorm/docs/2012/rxnorm_doco_full_2012-1.html (see section 6)",88 +DISPENSING,DISPENSE_SUP,RDBMS Number(8),Numeric(8),,NO,DAYS,,,"Days supply. Number of days that the medication supports based on the number of doses as reported by the pharmacist. This amount is typically found on the dispensing record. Integer values are expected. + +Important: Please do not calculate during CDM implementation. This field should only reflect originating source system calculations.",89 +DISPENSING,DISPENSE_AMT,RDBMS Number(8),Numeric(8),,NO,UNITS,,,"Number of units (pills, tablets, vials) dispensed. Net amount per NDC per dispensing. This amount is typically found on the dispensing record. Positive values are expected. + +Important: Please do not calculate during CDM implementation. This field should only reflect originating source system calculations. +",90 +DISPENSING,RAW_NDC,RDBMS Text(x),Numeric(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",91 +LAB_RESULT_CM ,LAB_RESULT_CM_ID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique LAB_RESULT_CM record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. + +(New field added to v3.0.)",92 +LAB_RESULT_CM ,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,93 +LAB_RESULT_CM ,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier used to link across tables. This is an optional field, and should only be populated if the lab was collected as part of a healthcare encounter.",94 +LAB_RESULT_CM ,LAB_NAME,RDBMS Text(10),SAS Char(10),,NO,,A1C;CK;CK_MB;CK_MBI;CREATININE;HGB;LDL;INR;TROP_I;TROP_T_QL;TROP_T_QN,"A1C=Hemoglobin A1c +CK=Creatine kinase total +CK_MB=Creatine kinase MB +CK_MBI=Creatine kinase MB/creatine kinase total +CREATININE=Creatinine +HGB=Hemoglobin +LDL=Low-density lipoprotein +INR=International normalized ratio +TROP_I=Troponin I cardiac +TROP_T_QL=Troponin T cardiac (qualitative) +TROP_T_QN=Troponin T cardiac (quantitative)","Laboratory result common measure, a categorical identification for the type of test, which is harmonized across all contributing data partners. + +Please note that it is possible for more than one LOINC code, CPT code, and/or local code to be associated with one LAB_NAME.",95 +LAB_RESULT_CM ,SPECIMEN_SOURCE,RDBMS Text(10),SAS Char(10),,NO,,BLOOD;CSF;PLASMA;PPP;SERUM;SR_PLS;URINE;NI;UN;OT,"BLOOD=blood +CSF=cerebrospinal fluid +PLASMA=plasma +PPP=platelet poor plasma +SERUM=serum +SR_PLS=serum/plasma +URINE=urine +NI=No information +UN=Unknown +OT=Other",Specimen source. All records will have a specimen source; some tests have several possible values for SPECIMEN_SOURCE. Please see the reference tables for additional details.,96 +LAB_RESULT_CM ,LAB_LOINC,RDBMS Text(10),SAS Char(10),,NO,,,,"Logical Observation Identifiers, Names, and Codes (LOINC) from the Regenstrief Institute. Results with local versions of LOINC codes (e.g., LOINC candidate codes) should be included in the RAW_table field, but the LOINC variable should be set to missing. Current LOINC codes are from 3-7 characters long but Regenstrief suggests a length of 10 for future growth. The last digit of the LOINC code is a check digit and is always preceded by a hyphen. All parts of the LOINC code, including the hyphen, must be included. Do not pad the LOINC code with leading zeros. + +Please see the LOINC reference table for known LOINC codes for each LAB_NAME. +",97 +LAB_RESULT_CM ,PRIORITY,RDBMS Text(2),SAS Char(2),,NO,,E;R;S;NI;UN;OT,"E=Expedite +R=Routine +S=Stat +NI=No information +UN=Unknown +OT=Other",Immediacy of test. The intent of this variable is to determine whether the test was obtained as part of routine care or as an emergent/urgent diagnostic test (designated as Stat or Expedite).,98 +LAB_RESULT_CM ,RESULT_LOC,RDBMS Text(2),SAS Char(2),,NO,,L;P;NI;UN;OT,"L=Lab +P=Point of Care +NI=No information +UN=Unknown +OT=Other","Location of the test result. Point of Care locations may include anticoagulation clinic, newborn nursery, finger stick in provider office, or home. The default value is ?L? unless the result is Point of Care. There should not be any missing values.",99 +LAB_RESULT_CM ,LAB_PX,RDBMS Text(11),SAS Char(11),,NO,,,,"Optional variable for local and standard procedure codes, used to identify the originating order for the lab test.",100 +LAB_RESULT_CM ,LAB_PX_TYPE,RDBMS Text(2),SAS Char(2),,NO,,09;10;11;C2;C3;C4;H3;HC;LC;ND;RE;NI;UN;OT,"09=ICD-9-CM +10=ICD-10-PCS +11=ICD-11-PCS +C2=CPT Category II +C3=CPT Category III +C4=CPT-4 (i.e., HCPCS Level I) +H3=HCPCS Level III +HC=HCPCS (i.e., HCPCS Level II) +LC=LOINC +ND=NDC +RE=Revenue +NI=No information +UN=Unknown +OT=Other","Procedure code type, if applicable.",101 +LAB_RESULT_CM ,LAB_ORDER_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date test was ordered.,102 +LAB_RESULT_CM ,SPECIMEN_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date specimen was collected.,103 +LAB_RESULT_CM ,SPECIMEN_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Time specimen was collected.,104 +LAB_RESULT_CM ,RESULT_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Result date.,105 +LAB_RESULT_CM ,RESULT_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Result time.,106 +LAB_RESULT_CM ,RESULT_QUAL,RDBMS Text(12),SAS Char(12),,NO,,"BORDERLINE;POSITIVE;NEGATIVE;UNDETERMINED;NI=No information;UN=Unknown;OT=Other +","BORDERLINE +POSITIVE +NEGATIVE +UNDETERMINED +NI=No information +UN=Unknown +OT=Other",Standardized result for qualitative results. This variable should be NI for quantitative results. Please see the reference tables for additional details and information on acceptable values for each qualitative LAB_NAME.,107 +LAB_RESULT_CM ,RESULT_NUM,RDBMS Number(8),SAS Char(8),,NO,,,,Standardized/converted result for quantitative results. This variable should be null for qualitative results. Please see the reference tables for additional details. (Modification of wording made from ?blank? to ?null? in v3.0).,108 +LAB_RESULT_CM ,RESULT_MODIFIER,RDBMS Text(2),SAS Char(2),,NO,,EQ;GE;GT;LE;LT;TX;NI;UN;OT,"EQ=Equal +GE=Greater than or equal to +GT=Greater than +LE=Less than or equal to +LT=Less than +TX=Text +NI=No information +UN=Unknown +OT=Other","Modifier for result values. Any symbols in the RAW_RESULT value should be reflected in the RESULT_MODIFIER variable. + +For example, if the original source data value is ""<=200"" then RAW_RESULT=200 and RESULT_MODIFIER=LE. If the original source data value is text then RESULT_MODIFIER=TX. If the original source data value is a numeric value then RESULT_MODIFIER=EQ.",109 +LAB_RESULT_CM ,RESULT_UNIT,RDBMS Text(11),SAS Char(11),,NO,,,,Converted/standardized units for the result. Please see the standard abbreviations reference table for additional details.,110 +LAB_RESULT_CM ,NORM_RANGE_LOW,RDBMS Text(10),SAS Char(10),,NO,,,,"Lower bound of the normal range assigned by the laboratory. Value should only contain the value of the lower bound. The symbols >, <, >=, <= should be removed. For example, if the normal range for a test is >100 and <300, then ""100"" should be entered.",111 +LAB_RESULT_CM ,NORM_MODIFIER_LOW,RDBMS Text(2),SAS Char(2),,NO,,EQ;GE;GT;NO;NI;UN;OT,"EQ=Equal +GE=Greater than or equal to +GT=Greater than +NO=No lower limit +NI=No information +UN=Unknown +OT=Other","Modifier for NORM_RANGE_LOW values. + +v3.0 modification made to field name. + +For numeric results one of the following needs to be true: +1) Both MODIFIER_LOW and MODIFIER_HIGH contain EQ (e.g. normal values fall in the range 3-10) +2) MODIFIER_LOW contains GT or GE and MODIFIER_HIGH contains NO (e.g. normal values are >3 with no upper boundary) +3) MODIFIER_HIGH contains LT or LE and MODIFIER_LOW contains NO (e.g. normal values are <=10 with no lower boundary) + +For numeric results one of the following needs to be true: +1) Both MODIFIER_LOW and MODIFIER_HIGH contain EQ (e.g. normal values fall in the range 3-10) +2) MODIFIER_LOW contains GT or GE and MODIFIER_HIGH contains NO (e.g. normal values are >3 with no upper boundary) +3) MODIFIER_HIGH contains LT or LE and MODIFIER_LOW contains NO (e.g. normal values are <=10 with no lower boundary)",112 +LAB_RESULT_CM ,NORM_RANGE_HIGH,RDBMS Text(10),SAS Char(10),,NO,,,,"Upper bound of the normal range assigned by the laboratory. Value should only contain the value of the upper bound. The symbols >, <, >=, <= should be removed. For example, if the normal range for a test is >100 and <300, then ""300"" should be entered.",113 +LAB_RESULT_CM ,NORM_MODIFIER_HIGH,RDBMS Text(2),SAS Char(2),,NO,,EQ;LE;LT;NO;NI;UN;OT,"EQ=Equal +LE=Less than or equal to +LT=Less than +NO=No higher limit +NI=No information +UN=Unknown +OT=Other","Modifier for NORM_RANGE_HIGH values. + +v3.0 modification made to field name. + +For numeric results one of the following needs to be true: +1) Both MODIFIER_LOW and MODIFIER_HIGH contain EQ (e.g. normal values fall in the range 3-10) +2) MODIFIER_LOW contains GT or GE and MODIFIER_HIGH contains NO (e.g. normal values are >3 with no upper boundary) +3) MODIFIER_HIGH contains LT or LE and MODIFIER_LOW contains NO (e.g. normal values are <=10 with no lower boundary)",114 +LAB_RESULT_CM ,ABN_IND,RDBMS Text(2),SAS Char(2),,NO,,AB;AH;AL;CH;CL;CR;IN;NL;NI;UN;OT,"AB=Abnormal +AH=Abnormally high +AL=Abnormally low +CH=Critically high +CL=Critically low +CR=Critical +IN=Inconclusive +NL=Normal +NI=No information +UN=Unknown +OT=Other",Abnormal result indicator. This value comes from the source data; do not apply logic to create it.,115 +LAB_RESULT_CM ,RAW_LAB_NAME,RDBMS Text(x),SAS Char(x),,NO,,,,"Local code related to an individual lab test. This variable will not be used in queries, but may be used by local programmers to associate a record with a particular LAB_NAME.",116 +LAB_RESULT_CM ,RAW_LAB_CODE,RDBMS Text(x),SAS Char(x),,NO,,,,"Local code related to an individual lab test. This variable will not be used in queries, but may be used by local programmers to associate a record with a particular LAB_NAME.",117 +LAB_RESULT_CM ,RAW_PANEL,RDBMS Text(x),SAS Char(x),,NO,,,,"Local code related to a battery or panel of lab tests. This variable will not be used in queries, but may be used by local programmers to associate a record with a particular LAB_NAME.",118 +LAB_RESULT_CM ,RAW_RESULT,RDBMS Text(x),SAS Char(x),,NO,,,,"The original test result value as seen in your source data. Values may include a decimal point, a sign or text (e.g., POSITIVE, NEGATIVE, DETECTED). The symbols >, <, >=, <= should be removed from the value and stored in the Modifier variable instead.",119 +LAB_RESULT_CM ,RAW_UNIT,RDBMS Text(x),SAS Char(x),,NO,,,,Original units for the result in your source data.,120 +LAB_RESULT_CM ,RAW_ORDER_DEPT,RDBMS Text(x),SAS Char(x),,NO,,,,Local code for ordering provider department.,121 +LAB_RESULT_CM ,RAW_FACILITY_CODE,RDBMS Text(x),SAS Char(x),,NO,,,,Local facility code that identifies the hospital or clinic. Taken from facility claims.,122 +CONDITION,CONDITIONID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",123 +CONDITION,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,124 +CONDITION,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier used to link across tables. This is an optional field, and should only be populated if the item was collected as part of a healthcare encounter. + +If more than one encounter association is present, this field should be populated with the ID of the encounter when the condition was first entered into the system. However, please note that many conditions may be recorded outside of an encounter context. (Guidance added in v3.0.)",125 +CONDITION,REPORT_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date condition was noted, which may be the date when it was recorded by a provider or nurse, or the date on which the patient reported it. Please note that this date may not correspond to onset date. (Additional guidance added in v3.0.)",126 +CONDITION,RESOLVE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date condition was resolved, if resolution of a transient condition has been achieved. A resolution date is not generally expected for chronic conditions, even if the condition is managed.",127 +CONDITION,ONSET_DATE,RDBMS Date,SAS Date (Numeric),,,,,,"New field added in v3.0. + +Please note that onset date is a very precise concept. Please do not map data unless they precisely match this definition. The REPORT_DATE concept may be a better fit for many systems.",128 +CONDITION,CONDITION_STATUS,RDBMS Text(2),SAS Char(2),,NO,,AC;RS;IN;NI;UN;OT,"AC=Active +RS=Resolved +IN=Inactive +NI=No information +UN=Unknown +OT=Other","Condition status corresponding with REPORT_DATE. + +Guidance: The value of IN=Inactive may be used in situations where a condition is not resolved, but is not currently active (for example, psoriasis).",129 +CONDITION,CONDITION,RDBMS Text(18),SAS Char(18),,NO,,,,"Condition code. + +Leading zeroes and different levels of decimal precision are permissible in this field. Please populate the exact textual value of this diagnosis code, but remove source-specific suffixes and prefixes. Other codes should be listed as recorded in the source data.",130 +CONDITION,CONDITION_TYPE,RDBMS Text(2),SAS Char(2),,NO,,09;10;11;SM;HP;AG;NI;UN;OT,"09=ICD-9-CM +10=ICD-10-CM +11=ICD-11-CM +SM=SNOMED CT +HP=Human Phenotype Ontology +AG=Algorithmic +NI=No information +UN=Unknown +OT=Other","Condition code type. + +Please note: The ?Other? category is meant to identify internal use ontologies and codes. + +v3.0 amendment: The new categorical value of AG has been added. ",131 +CONDITION,CONDITION_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,PR;HC;RG;PC;NI;UN;OT,"PR=Patient-reported medical history +HC=Healthcare problem list +RG=Registry cohort +PC=PCORnet-defined condition algorithm +NI=No information +UN=Unknown +OT=Other","Please note: The ?Patient-reported? category can include reporting by a proxy, such as patient?s family or guardian. + +Guidance: ?Registry cohort? generally refers to cohorts of patients flagged with a certain set of characteristics for management within a health system. + +?Patient-reported? can include self-reported medical history and/or current medical conditions, not captured via healthcare problem lists or registry cohorts. + +v3.0 amendment: The new categorical value of PC has been added.",132 +CONDITION,RAW_CONDITION_STATUS,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",133 +CONDITION,RAW_CONDITION,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",134 +CONDITION,RAW_CONDITION_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",135 +CONDITION,RAW_CONDITION_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",136 +PRO_CM,PRO_CM_ID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",137 +PRO_CM,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier for the patient for whom the PRO response was captured. Used to link across tables.,138 +PRO_CM,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier used to link across tables. This is an optional field, and should only be populated if the item was collected as part of a healthcare encounter.",139 +PRO_CM,PRO_ITEM,RDBMS Text(7),SAS Char(7),,NO,,PN_0001;PN_0002;PN_0003;PN_0004;PN_0005;PN_0006;PN_0007;PN_0008;PN_0009;PN_0010;PN_0011;PN_0012;PN_0013;PN_0014;PN_0015;PN_0016;PN_0017;PN_0018;PN_0019;PN_0020;PN_0021,"PN_0001=GLOBAL01 +PN_0002=GLOBAL02 +PN_0003=GLOBAL06 +PN_0004=PFA53 +PN_0005=EDDEP29 +PN_0006=HI7 +PN_0007=SLEEP20 +PN_0008=SRPPER11_CAPS +PN_0009=PAININ9 +PN_0010=3793R1 +PN_0011=28676R1 +PN_0012=EOS_P_011 +PN_0013=PEDSGLOBAL2 +PN_0014=PEDSGLOBAL5 +PN_0015=PEDSGLOBAL6 +PN_0016=GLOBAL03 +PN_0017=GLOBAL04 +PN_0018=EDANX53 +PN_0019=SAMHSA +PN_0020=CAHPS 4.0 +PN_0021=PA070",PCORnet identifier for the specific Common Measure item. Please see the Common Measures reference table for more details.,140 +PRO_CM,PRO_LOINC,RDBMS Text(10),SAS Char(10),,NO,,,,"LOINC code for item context and stem. Please see the reference table for known LOINC codes for each common measure. + +Logical Observation Identifiers, Names, and Codes (LOINC) from the Regenstrief Institute. Results with local versions of LOINC codes (e.g., LOINC candidate codes) should be included in the RAW_table field, but the PRO_LOINC variable should be set to missing. Current LOINC codes are from 3-7 characters long but Regenstrief suggests a length of 10 for future growth. The last digit of the LOINC code is a check digit and is always preceded by a hyphen. All parts of the LOINC code, including the hyphen, must be included. Do not pad the LOINC code with leading zeros. +",141 +PRO_CM,PRO_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,The date of the response.,142 +PRO_CM,PRO_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,The time of the response.,143 +PRO_CM,PRO_RESPONSE,NUMBER(8),Numeric(8),,NO,,,,The numeric response recorded for the item. Please see the Common Measures reference table for the list of valid responses for each item.,144 +PRO_CM,PRO_METHOD,RDBMS Text(2),SAS Char(2),,NO,,PA;EC;PH;IV;NI;UN;OT,"PA=Paper +EC=Electronic +PH=Telephonic +IV=Telephonic with interactive voice response (IVR) technology +NI=No information +UN=Unknown +OT=Other","Method of administration. Electronic includes responses captured via a personal or tablet computer, at web kiosks, or via a smartphone.",145 +PRO_CM,PRO_MODE,RDBMS Text(2),SAS Char(2),,NO,,"SF;SA;PR;PA;NI;UN;OT +","SF=Self without assistance +SA= Self with assistance +PR=Proxy without assistance +PA=Proxy with assistance +NI=No information +UN=Unknown +OT=Other","The person who responded on behalf of the patient for whom the response was captured. A proxy report is a measurement based on a report by someone other than the patient reporting as if he or she is the patient, such as a parent responding for a child, or a caregiver responding for an individual unable to report for themselves. Assistance excludes providing interpretation of the patient?s response. ",146 +PRO_CM,PRO_CAT,RDBMS Text(2),SAS Char(2),,NO,,"Y;N;NI;UN;OT +","Y=Yes +N=No +NI=No information +UN=Unknown +OT=Other",Indicates whether Computer Adaptive Testing (CAT) was used to administer the survey or instrument that the item was part of. May apply to electronic (EC) and telephonic (PH or IV) modes. ,147 +PRO_CM,RAW_PRO_CODE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating code, such as LOINC candidate codes that have not yet been adopted",148 +PRO_CM,RAW_PRO_RESPONSE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",149 +PRESCRIBING,PRESCRIBINGID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary identifier for each unique PRESCRIBING record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID.",150 +PRESCRIBING,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,151 +PRESCRIBING,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. This is an optional relationship; the ENCOUNTERID should be present if the prescribing activity is directly associated with an encounter.,152 +PRESCRIBING,RX_PROVIDERID,RDBMS Text(x),SAS Char(x),,NO,,,,Provider code for the provider who prescribed the medication. The provider code is a pseudoidentifier with a consistent crosswalk to the real identifier. ,153 +PRESCRIBING,RX_ORDER_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Order date of the prescription by the provider.,154 +PRESCRIBING,RX_ORDER_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,,,,Order time of the prescription by the provider.,155 +PRESCRIBING,RX_START_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Start date of order. This attribute may not be consistent with the date on which the patient actually begin taking the medication.,156 +PRESCRIBING,RX_END_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,End date of order (if available).,157 +PRESCRIBING,RX_QUANTITY,RDBMS Number(8),Numeric(8),,NO,,,,Quantity ordered.,158 +PRESCRIBING,RX_REFILLS,RDBMS Number(8),Numeric(8),,NO,,,,"Number of refills ordered (not including the original prescription). If no refills are ordered, the value should be zero.",159 +PRESCRIBING,RX_DAYS_SUPPLY,RDBMS Number(8),Numeric(8),,NO,,,,"Number of days supply ordered, as specified by the prescription.",160 +PRESCRIBING,RX_FREQUENCY,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;05;06;07;08;09;NI;UN;OT,"01=Every day +02=Two times a day (BID) +03=Three times a day (TID) +04=Four times a day (QID) +05=Every morning +06=Every afternoon +07=Before meals +08=After meals +09=As needed (PRN) +NI=No information +UN=Unknown +OT=Other +",Specified frequency of medication.,161 +PRESCRIBING,RX_BASIS,RDBMS Text(2),SAS Char(2),,NO,,01;02;NI;UN;OT,"01=Dispensing +02=Administration +NI=No information +UN=Unknown +OT=Other +",Basis of the medication order,162 +PRESCRIBING,RXNORM_CUI,RDBMS Number (8),Numeric(8),,NO,,,,"Where an RxNorm mapping exists for the source medication, this field contains the RxNorm concept identifier (CUI) at the highest possible specificity. + +If more than one option exists for mapping, the following ordered strategy may be adopted: +1)Semantic generic clinical drug +2)Semantic Branded clinical drug +3)Generic drug pack +4)Branded drug pack +",163 +PRESCRIBING,RAW_RX_MED_NAME,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating, full textual medication name from the source.",164 +PRESCRIBING,RAW_RX_FREQUENCY,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",165 +PRESCRIBING,RAW_RXNORM_CUI,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",166 +PCORNET_TRIAL,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,167 +PCORNET_TRIAL,TRIALID,RDBMS Text(20),SAS Char(20),,NO,,,,Each TRIALID is assigned by the PCORnet trial?s coordinating center.,168 +PCORNET_TRIAL,PARTICIPANTID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary person-level identifier used to uniquely identify a participant in a PCORnet trial. + +PARTICIPANTID is never repeated or reused for a specific clinical trial, and is generally assigned by trial-specific processes. It may be the same as a randomization ID. +",169 +PCORNET_TRIAL,TRIAL_SITEID,RDBMS Text(x),SAS Char(x),,NO,,,,Each TRIAL_SITEID is assigned by the PCORnet trial coordinating center.,170 +PCORNET_TRIAL,TRIAL_ENROLL_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Date on which the participant enrolled in the trial (generally coincides with trial consent process).,171 +PCORNET_TRIAL,TRIAL_END_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Date on which the participant completes participation in the trial.,172 +PCORNET_TRIAL,TRIAL_WITHDRAW_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,"If applicable, date on which the participant withdraws consent from the trial.",173 +PCORNET_TRIAL,TRIAL_INVITE_CODE,RDBMS Text(20),SAS Char(20),,NO,,,,"Textual strings used to uniquely identify invitations sent to potential participants, and allows acceptances to be associated back to the originating source. + +Where used, there should generally be a unique combination of PATID, TRIAL_NAME, and INVITE_CODE within each datamart. + +For example, this might include ?co-enrollment ID strings? for e-mail invites or ?verification codes? for letter invites. +",174 +DEATH,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,175 +DEATH,DEATH_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Date of death.,176 +DEATH,DEATH_DATE_IMPUTE,RDBMS Text(2),SAS Char(2),,NO,,B;D;M;N;NI;UN;OT,"B=Both month and day imputed +D=Day imputed +M=Month imputed +N=Not imputed +NI=No information +UN=Unknown +OT=Other +","When date of death is imputed, this field indicates which parts of the date were imputed.",177 +DEATH,DEATH_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,L;N;D;S;T;NI;UN;OT,"L=Other, locally defined +N=National Death Index +D=Social Security +S=State Death files +T=Tumor data +NI=No information +UN=Unknown +OT=Other +","Guidance: ?Other, locally defined? may be used to indicate presence of deaths reported from EHR systems, such as in-patient hospital deaths or dead on arrival.",178 +DEATH,DEATH_MATCH_CONFIDENCE,RDBMS Text(2),SAS Char(2),,NO,,E;F;P;NI;UN;OT,"E=Excellent +F=Fair +P=Poor +NI=No information +UN=Unknown +OT=Other +","For situations where a probabilistic patient matching strategy is used, this field indicates the confidence that the patient drawn from external source data represents the actual patient. + +Should not be present where DEATH_SOURCE is L (locally-defined). May not be applicable for DEATH_SOURCE=T (tumor registry data) +",179 +DEATH_CAUSE,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,180 +DEATH_CAUSE,DEATH_CAUSE,RDBMS Text(8),SAS Char(8),,NO,,,,Cause of death code. Please include the decimal point in ICD codes (if any).,181 +DEATH_CAUSE,DEATH_CAUSE_CODE,RDBMS Text(2),SAS Char(2),,NO,,09;10;NI;UN;OT,"09=ICD-9 +10=ICD-10 +NI=No information +UN=Unknown +OT=Other +",Cause of death code type.,182 +DEATH_CAUSE,DEATH_CAUSE_TYPE,RDBMS Text(2),SAS Char(2),,NO,,C;I;O;U;NI;UN;OT,"C=Contributory +I=Immediate/Primary +O=Other +U=Underlying +NI=No information +UN=Unknown +OT=Other +",Cause of death type. There should be only one underlying cause of death.,183 +DEATH_CAUSE,DEATH_CAUSE_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,L;N;D;S;T;NI;UN;OT,"L=Other, locally defined +N=National Death Index +D=Social Security +S=State Death files +T=Tumor data +NI=No information +UN=Unknown +OT=Other +","Source of cause of death information. + +Guidance: ?Other, locally defined? may be used to indicate presence of deaths reported from EHR systems, such as in-patient hospital deaths or dead on arrival. +",184 +DEATH_CAUSE,DEATH_CAUSE_CONFIDENCE,RDBMS Text(2),SAS Char(2),,NO,,E;F;P;NI;UN;OT,"E=Excellent +F=Fair +P=Poor +NI=No information +UN=Unknown +OT=Other +","Confidence in the accuracy of the cause of death based on source, match, number of reporting sources, discrepancies, etc.",185 +HARVEST,NETWORKID,RDBMS Text(10),SAS Char(10),,NO,,,,This identifier is assigned by DSSNI operations,186 +HARVEST,NETWORK_NAME,RDBMS Text(20),SAS Char(20),,NO,,,,Descriptive name of the network,187 +HARVEST,DATAMARTID,RDBMS Text(10),SAS Char(10),,NO,,,,This identifier is assigned by DSSNI operations,188 +HARVEST,DATAMART_NAME,RDBMS Text(20),SAS Char(20),,NO,,,,Descriptive name of the datamart,189 +HARVEST,DATAMART_PLATFORM,RDBMS Text(2),SAS Char(2),,NO,,"01;02;03;04;05;NI;UN;OT +","01=SQL Server +02=Oracle +03=PostgreSQL +04=MySQL +05=SAS +NI=No information +UN=Unknown +OT=Other +",,190 +HARVEST,CDM_VERSION,NUMBER(8),Numeric(8),,NO,,,,"Version currently implemented within this datamart (for example, 1.0, 2.0, 3.0).",191 +HARVEST,DATAMART_CLAIMS,RDBMS Text(2),SAS Char(2),,NO,,01;02;NI;UN;OT,"01=Not present +02=Present +NI=No information +UN=Unknown +OT=Other +",Datamart includes claims data source(s),192 +HARVEST,DATAMART_EHR,RDBMS Text(2),SAS Char(2),,NO,,01;02;NI;UN;OT,"01=Not present +02=Present +NI=No information +UN=Unknown +OT=Other +",Datamart includes EHR data source(s),193 +HARVEST,BIRTH_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the BIRTH_DATE field on the DEMOGRAPHIC table + +Please see notes for additional definitions. +",194 +HARVEST,ENR_START_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the ENR_START_DATE field on the ENROLLMENT table + +Please see notes for additional definitions. +",195 +HARVEST,ENR_END_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the ENR_END_DATE field on the ENROLLMENT table + +Please see notes for additional definitions. +",196 +HARVEST,ADMIT_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the ADMIT_DATE field on the ENCOUNTER table + +Please see notes for additional definitions. +",197 +HARVEST,DISCHARGE_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the DISCHARGE_DATE field on the ENCOUNTER table + +Please see notes for additional definitions. +",198 +HARVEST,PX_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the PX_DATE field on the PROCEDURES table + +Please see notes for additional definitions. +",199 +HARVEST,RX_ORDER_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the RX_ORDER_DATE field on the PRESCRIBING table + +Please see notes for additional definitions. +",200 +HARVEST,RX_START_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the RX_START_DATE field on the PRESCRIBING table + +Please see notes for additional definitions. +",201 +HARVEST,RX_END_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the RX_END_DATE field on the PRESCRIBING table + +Please see notes for additional definitions. +",202 +HARVEST,DISPENSE_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the DISPENSE_DATE field on the DISPENSING table + +Please see notes for additional definitions. +",203 +HARVEST,LAB_ORDER_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the LAB_ORDER_DATE field on the LAB_RESULT_CM table + +Please see notes for additional definitions. +",204 +HARVEST,SPECIMEN_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the SPECIMEN_DATE field on the LAB_RESULT_CM table + +Please see notes for additional definitions. +",205 +HARVEST,RESULT_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the RESULT_DATE field on the LAB_RESULT_CM table + +Please see notes for additional definitions. +",206 +HARVEST,MEASURE_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the MEASURE_DATE field on the VITAL table + +Please see notes for additional definitions. +",207 +HARVEST,ONSET_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the ONSET_DATE field on the CONDITION table + +Please see notes for additional definitions. +",208 +HARVEST,REPORT_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the REPORT_DATE field on the CONDITION table + +Please see notes for additional definitions. +",209 +HARVEST,RESOLVE_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the RESOLVE_DATE field on the CONDITION table + +Please see notes for additional definitions. +",210 +HARVEST,PRO_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the PRO_DATE field on the PRO_CM table + +Please see notes for additional definitions. +",211 +HARVEST,REFRESH_DEMOGRAPHIC_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DEMOGRAPHIC table. This date should be null if the table does not have records.,212 +HARVEST,REFRESH_ENROLLMENT_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the ENROLLMENT table. This date should be null if the table does not have records.,213 +HARVEST,REFRESH_ENCOUNTER_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the ENCOUNTER table. This date should be null if the table does not have records.,214 +HARVEST,REFRESH_DIAGNOSIS_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DIAGNOSIS table. This date should be null if the table does not have records.,215 +HARVEST,REFRESH_PROCEDURES_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PROCEDURES table. This date should be null if the table does not have records.,216 +HARVEST,REFRESH_VITAL_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the VITAL table. This date should be null if the table does not have records.,217 +HARVEST,REFRESH_DISPENSING_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DISPENSING table. This date should be null if the table does not have records.,218 +HARVEST,REFRESH_LAB_RESULT_CM_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the LAB_RESULT_CM table. This date should be null if the table does not have records.,219 +HARVEST,REFRESH_CONDITION_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the CONDITION table. This date should be null if the table does not have records.,220 +HARVEST,REFRESH_PRO_CM_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PRO_CM table. This date should be null if the table does not have records.,221 +HARVEST,REFRESH_PRESCRIBING_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PRESCRIBING table. This date should be null if the table does not have records.,222 +HARVEST,REFRESH_PCORNET_TRIAL_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PCORNET_TRIAL table. This date should be null if the table does not have records.,223 +HARVEST,REFRESH_DEATH_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DEATH table. This date should be null if the table does not have records.,224 +HARVEST,REFRESH_DEATH_CAUSE_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DEATH_CAUSE table. This date should be null if the table does not have records.,225 From cca54f7a1700bf204d69142f74d143e8fa807309 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 5 Dec 2016 16:25:39 -0600 Subject: [PATCH 190/507] Get the list of CDM tables to backup from the specification (4407) --- Oracle/backup_cdm.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Oracle/backup_cdm.py b/Oracle/backup_cdm.py index de8ef87..c58f089 100644 --- a/Oracle/backup_cdm.py +++ b/Oracle/backup_cdm.py @@ -1,14 +1,20 @@ ''' backup_cdm - back up cdm tables that have at least one row ''' +import csv +import pkg_resources as pkg + + # Allow --dry run even without cx_Oracle try: from cx_Oracle import DatabaseError except: pass -CDM3_TABLES = ['HARVEST', 'DEMOGRAPHIC', 'ENCOUNTER', 'DIAGNOSIS', - 'CONDITION', 'PROCEDURES', 'VITAL', 'ENROLLMENT', - 'LAB_RESULT_CM', 'PRESCRIBING', 'DISPENSING'] +CDM_SPEC = '../2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv' # noqa + +CDM3_TABLES = sorted(set(f['TABLE_NAME'] + for f in csv.DictReader( + pkg.resource_stream(__name__, CDM_SPEC)))) def main(get_cursor): From 8d0140fc11fcb2862fe1e585ec5a1d4fcc3a569f Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 5 Dec 2016 17:17:32 -0600 Subject: [PATCH 191/507] fix providerid too --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 50c5630..ccb5a9b 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1036,7 +1036,7 @@ with icd10_transition as ( where diag.c_fullname like '\PCORI\DIAGNOSIS\%' ) -select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, enc.admit_date, factline.provider_id +select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, enc.admit_date, enc.providerid , case when diag.pcori_basecode is not null then diag.pcori_basecode when enc.admit_date >= icd10_transition.cutoff then diag.icd10_code From cea3c903ceaa81dc941b6faef156a4021450a8db Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 5 Dec 2016 17:17:48 -0600 Subject: [PATCH 192/507] don't hard code blueheronmetadata Thanks, Nathan, for review. --- Oracle/PCORNetLoader_ora.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index ccb5a9b..1ebe2cc 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1016,12 +1016,12 @@ with icd10_transition as ( /* DX_IDs may have mappings to both ICD9 and ICD10 */ , has9 as ( select distinct c_basecode, pcori_basecode icd9_code - from blueheronmetadata.pcornet_diag diag + from "&&i2b2_meta_schema".pcornet_diag diag where diag.c_fullname like '\PCORI\DIAGNOSIS\09%' ) , has10 as ( select distinct c_basecode, pcori_basecode icd10_code - from blueheronmetadata.pcornet_diag diag + from "&&i2b2_meta_schema".pcornet_diag diag where diag.c_fullname like '\PCORI\DIAGNOSIS\10%' ) , has9_and_10 as ( From 2fdf1a8e16d2aaa7c431acf009544afbe6b73ec8 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 5 Dec 2016 17:18:52 -0600 Subject: [PATCH 193/507] prune obsolete to_char() etc. restore clarity TODO Thanks for review, Nathan --- Oracle/pcornet_mapping.sql | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index ada9c60..9debde7 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -102,7 +102,7 @@ update "&&i2b2_meta_schema".pcornet_diag pd set pd.pcori_basecode = 'PDX:X' wher the pcornet_basecode with the expected values. */ --- select * +--select * delete from "&&i2b2_meta_schema".PCORNET_DIAG where c_fullname like '\PCORI\DIAGNOSIS\09\%' @@ -111,6 +111,7 @@ where c_fullname like '\PCORI\DIAGNOSIS\09\%' insert into "&&i2b2_meta_schema".PCORNET_DIAG +-- TODO: Stop cheating by going back to Clarity with edg9 as ( select dx_id, code from clarity.edg_current_icd9@id ) @@ -121,6 +122,7 @@ with edg9 as ( select case when ht.c_basecode like 'KUH|DX_ID:%' then to_number(replace(ht.c_basecode, 'KUH|DX_ID:', '')) + else null end dx_id , ht.* from "&&i2b2_meta_schema"."&&terms_table" ht @@ -140,9 +142,9 @@ with edg9 as ( , ht.* from heron_dx ht left join edg9 - on to_char(edg9.dx_id) = ht.dx_id + on edg9.dx_id = ht.dx_id left join edg10 - on to_char(edg10.dx_id) = ht.dx_id + on edg10.dx_id = ht.dx_id ) select td.c_hlevel, From 42d400ea11359a5b549082b4cfa6370e40276e61 Mon Sep 17 00:00:00 2001 From: Nathan Graham Date: Mon, 5 Dec 2016 18:56:50 -0600 Subject: [PATCH 194/507] Back up CDM tables even if they have 0 rows (#4407) (#51) - Print to stderr as it's flushed more frequently and we'll get better real-time status updates during the backup. --- Oracle/backup_cdm.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/Oracle/backup_cdm.py b/Oracle/backup_cdm.py index c58f089..84c8266 100644 --- a/Oracle/backup_cdm.py +++ b/Oracle/backup_cdm.py @@ -1,8 +1,10 @@ ''' backup_cdm - back up cdm tables that have at least one row ''' +from __future__ import print_function + import csv import pkg_resources as pkg - +from sys import stderr # Allow --dry run even without cx_Oracle try: @@ -24,20 +26,24 @@ def main(get_cursor): bak_schema_table = ('%(backup_schema)s.%(table)s' % dict(backup_schema=backup_schema, table=table)) - rcount = count_rows(cursor, table) - print ('%(table)s (to be backed up) currently has %(rcount)s rows.' - % dict(table=table, rcount=rcount)) + eprint('%(table)s (to be backed up) currently has %(rcount)s rows.' + % dict(table=table, rcount=count_rows(cursor, table))) + + drop(cursor, bak_schema_table) + copy(cursor, table, bak_schema_table) + eprint('%(bak_schema_table)s now has %(rows)s rows.' % + dict(bak_schema_table=bak_schema_table, + rows=count_rows(cursor, bak_schema_table))) + - if rcount: - drop(cursor, bak_schema_table) - copy(cursor, table, bak_schema_table) - print ('%(bak_schema_table)s now has %(rows)s rows.' % - dict(bak_schema_table=bak_schema_table, - rows=count_rows(cursor, bak_schema_table))) +def eprint(*args, **kwargs): + ''' ACK: http://stackoverflow.com/questions/5574702/how-to-print-to-stderr-in-python # noqa + ''' + print(*args, file=stderr, **kwargs) def drop(cursor, schema_table): - print ('Dropping %(schema_table)s (%(rows)s rows).' % + eprint('Dropping %(schema_table)s (%(rows)s rows).' % dict(schema_table=schema_table, rows=count_rows(cursor, schema_table))) try: @@ -60,7 +66,7 @@ def count_rows(cursor, schema_table): def copy(cursor, table, bak_schema_table): - print ('Copying %(table)s to %(bak_schema_table)s.' % + eprint('Copying %(table)s to %(bak_schema_table)s.' % dict(table=table, bak_schema_table=bak_schema_table)) cursor.execute('create table %(bak_schema_table)s as ' 'select * from %(table)s' % @@ -72,12 +78,12 @@ def __init__(self): self.fetch_count = 1 def execute(self, sql): - print 'execute: ' + sql + eprint('execute: ' + sql) def fetchall(self): r = [(self.fetch_count, )] self.fetch_count += 1 - print 'fetch returning: ' + str(r) + eprint('fetch returning: ' + str(r)) return r if __name__ == '__main__': From d15d7cc46070700299cd3c4a37d4e5455240898d Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 6 Dec 2016 08:32:49 -0600 Subject: [PATCH 195/507] exclude null pcori_basecode in has9 and has10 (#52) --- Oracle/PCORNetLoader_ora.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 1ebe2cc..66ee398 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1018,11 +1018,13 @@ with icd10_transition as ( select distinct c_basecode, pcori_basecode icd9_code from "&&i2b2_meta_schema".pcornet_diag diag where diag.c_fullname like '\PCORI\DIAGNOSIS\09%' + and pcori_basecode is not null ) , has10 as ( select distinct c_basecode, pcori_basecode icd10_code from "&&i2b2_meta_schema".pcornet_diag diag where diag.c_fullname like '\PCORI\DIAGNOSIS\10%' + and pcori_basecode is not null ) , has9_and_10 as ( select has9.c_basecode, has9.icd9_code, has10.icd10_code From e09eefe47eee4e765597e5ee5a2c194a9180af3f Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 6 Dec 2016 09:51:42 -0600 Subject: [PATCH 196/507] PCORNetDeath from SCILHS upstream commit: d2681db Sep 1 --- Oracle/PCORNetLoader_ora.sql | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 66ee398..e4f9885 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1804,9 +1804,31 @@ end PCORNetDispensing; / +---------------------------------------------------------------------------------------------------------------------------------------- +---------------------------------------------------------------------------------------------------------------------------------------- +-- 11. Death - by Jeff Klann, PhD +-- Simple transform only pulls death date from patient dimension, does not rely on an ontology +-- Translated from MSSQL script by Matthew Joss +---------------------------------------------------------------------------------------------------------------------------------------- +create or replace procedure PCORNetDeath as + +begin + +insert into pmndeath( patid, death_date, death_date_impute, death_source, death_match_confidence) +select distinct pat.patient_num, pat.death_date, case +when vital_status_cd like 'X%' then 'B' +when vital_status_cd like 'M%' then 'D' +when vital_status_cd like 'Y%' then 'N' +else 'OT' +end, 'NI','NI' +from i2b2patient pat +where (pat.death_date is not null or vital_status_cd like 'Z%') and pat.patient_num in (select patid from pmndemographic); + +end; +/ create or replace PROCEDURE pcornetReport @@ -1906,6 +1928,7 @@ http://listserv.kumc.edu/pipermail/gpc-dev/attachments/20160223/8d79fa70/attachm > what we have in our i2b2 */ PCORNetDispensing; +PCORNetDeath; PCORNetHarvest; end pcornetloader; From 72b46837729076b27f3681f118db8583e10735f8 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 6 Dec 2016 10:02:53 -0600 Subject: [PATCH 197/507] drop death code from cdm_postproc ...after adopting SCILHS DEATH code to KUMC naming: - we don't preface our CDM tables with `pmn` (I forget how we diverged) - our vital_status_cd is lower-case --- Oracle/PCORNetLoader_ora.sql | 6 +++--- Oracle/cdm_postproc.sql | 11 ----------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index e4f9885..e78b66f 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1817,15 +1817,15 @@ create or replace procedure PCORNetDeath as begin -insert into pmndeath( patid, death_date, death_date_impute, death_source, death_match_confidence) +insert into death( patid, death_date, death_date_impute, death_source, death_match_confidence) select distinct pat.patient_num, pat.death_date, case when vital_status_cd like 'X%' then 'B' when vital_status_cd like 'M%' then 'D' -when vital_status_cd like 'Y%' then 'N' +when upper(vital_status_cd) like 'Y%' then 'N' else 'OT' end, 'NI','NI' from i2b2patient pat -where (pat.death_date is not null or vital_status_cd like 'Z%') and pat.patient_num in (select patid from pmndemographic); +where (pat.death_date is not null or vital_status_cd like 'Z%') and pat.patient_num in (select patid from demographic); end; / diff --git a/Oracle/cdm_postproc.sql b/Oracle/cdm_postproc.sql index 9324c2d..27790b2 100644 --- a/Oracle/cdm_postproc.sql +++ b/Oracle/cdm_postproc.sql @@ -25,14 +25,3 @@ The CDM wants height in inches and weight in pounds. */ update vital v set v.ht = v.ht / 2.54; update vital v set v.wt = v.wt / 16; -/* Populate death table. Eventually, we expect this to be added to the upstream -transform code. -Ref: https://github.com/SCILHS/i2p-transform/issues/3 -*/ -insert into death -select - pd.patient_num, pd.death_date, 'N' death_date_impute, - 'UN' death_source, -- TODO: We have SSMDF and EMR sources at least - 'E' death_match_confidence -from "&&i2b2_data_schema".patient_dimension pd -where pd.vital_status_cd = 'y'; From c5ad67c49def9c9a64f3f83625d708206590edb3 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 7 Dec 2016 00:40:52 -0600 Subject: [PATCH 198/507] handle null pcori_basecode in pcornet_diag --- Oracle/PCORNetLoader_ora.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index e78b66f..9d97b47 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1036,6 +1036,7 @@ with icd10_transition as ( from pcornet_diag diag left join has9_and_10 on has9_and_10.c_basecode = diag.c_basecode where diag.c_fullname like '\PCORI\DIAGNOSIS\%' + and diag.pcori_basecode is not null ) select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, enc.admit_date, enc.providerid From 3d48c1b27e9e24279d5d57df219a5cb7de232a4c Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 7 Dec 2016 15:13:46 -0600 Subject: [PATCH 199/507] Remove MAR and inpatient dispensing facts from the PRESCRIBING modifier mapping. --- Oracle/pcornet_mapping.csv | 2 -- 1 file changed, 2 deletions(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index f4cb990..d380049 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -4,8 +4,6 @@ pcori_path,local_path \PCORI_MOD\RX_BASIS\PR\OT\,\Medication\Other Orders\ \PCORI_MOD\RX_BASIS\PR\01\,\Medication\Outpatient\ \PCORI_MOD\RX_BASIS\PR\02\,\Medication\PRN Inpatient Order\ -\PCORI_MOD\RX_BASIS\PR\01\,\Medication\Dispensed\ -\PCORI_MOD\RX_BASIS\PR\OT\,\Medication\MAR Result\ \PCORI_MOD\PDX\P\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\discharge_primary\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\DI\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\AD\,\Diagnosis\\modifier_dx\billing_diagnosis\admit\ From be2a8b85a398bdc6afd28d58cf75bd106a3adaa2 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Dec 2016 12:38:55 -0600 Subject: [PATCH 200/507] Replaced left joins on modifier_cd with instance_num in order to merge facts --- Oracle/PCORNetLoader_ora.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 9d97b47..df58bba 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1657,35 +1657,35 @@ inner join encounter enc on enc.encounterid = m.encounter_Num and m.concept_cd = basis.concept_Cd and m.start_date = basis.start_date and m.provider_id = basis.provider_id - and m.modifier_cd = basis.modifier_cd + and m.instance_num = basis.instance_num left join freq on m.encounter_num = freq.encounter_num and m.concept_cd = freq.concept_Cd and m.start_date = freq.start_date and m.provider_id = freq.provider_id - and m.modifier_cd = freq.modifier_cd + and m.instance_num = freq.instance_num left join quantity on m.encounter_num = quantity.encounter_num and m.concept_cd = quantity.concept_Cd and m.start_date = quantity.start_date and m.provider_id = quantity.provider_id - and m.modifier_cd = quantity.modifier_cd + and m.instance_num = quantity.instance_num left join refills on m.encounter_num = refills.encounter_num and m.concept_cd = refills.concept_Cd and m.start_date = refills.start_date and m.provider_id = refills.provider_id - and m.modifier_cd = refills.modifier_cd + and m.instance_num = refills.instance_num left join supply on m.encounter_num = supply.encounter_num and m.concept_cd = supply.concept_Cd and m.start_date = supply.start_date and m.provider_id = supply.provider_id - and m.modifier_cd = supply.modifier_cd + and m.instance_num = supply.instance_num where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); From 32e2d9f8bab0c8969078a9317aa50771393668cc Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Dec 2016 12:40:16 -0600 Subject: [PATCH 201/507] Replace deid instance_nums with deid order_id to allow for merger of med facts --- Oracle/observation_fact_meds.sql | 61 ++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/Oracle/observation_fact_meds.sql b/Oracle/observation_fact_meds.sql index 0fe9f95..b5b3708 100644 --- a/Oracle/observation_fact_meds.sql +++ b/Oracle/observation_fact_meds.sql @@ -63,3 +63,64 @@ create bitmap index MED_OBS_FACT_NVAL_NUM_BI create bitmap index MED_OBS_FACT_MOD_CODE_BI on observation_fact_meds (MODIFIER_CD) nologging parallel 6; + +/* + * NOTE: The below code is a fix for Cycle 2 which replaces deid instance_nums + * with deid order_ids in i2b2medfacts inorder to allow the SCILHS PRESCRIBING + * code to merge facts correctly (see PCORNetPrescribing in + * PCORNetLoader_ora.sql). + */ + + -- Please excuse the copy and paste from i2b2_facts_deid.sql, but this is + -- needed for the mapping below (portions of which are also copied from + -- i2b2_facts_deid.sql). + create or replace view byte_conv as + select num_bytes, rpad('x', num_bytes * 2, 'x') as xfill + from (select 7 as num_bytes from dual); + +-- Generate deid instance_num order_id mapping +drop table med_order_instance_map; + +create table med_order_instance_map as +with id_med_instance as ( + select instance_num + from nightherondata.observation_fact@id + where concept_cd like 'KUH|MEDICATION_ID:%' +), +-- based on instance_num deid in i2b2_facts_deid.sql. +order_id_mapping as ( + select + to_number(substr(instance_num, -1)) val, + case when to_number(substr(instance_num, -1)) < 3 then floor(instance_num / 1e5) + else floor(instance_num / 1e6) end order_id, -- Corrects for bug in HERON ETL + instance_num, + to_number( + rawtohex( + UTL_RAW.SUBSTR( + DBMS_CRYPTO.Hash(UTL_RAW.CAST_FROM_NUMBER(instance_num), + /* typ 3 is HASH_SH1 */ + 3), + 1, (select num_bytes from byte_conv))), + (select xfill from byte_conv)) deid_instance_num + from id_med_instance +), +distinct_order_ids as ( + select distinct order_id from order_id_mapping +), +deid_id_order_id_mapping as ( + select rownum deid_order_id, order_id + from distinct_order_ids +) +select distinct dim.deid_order_id, oim.deid_instance_num +from order_id_mapping oim +join deid_id_order_id_mapping dim + on oim.order_id=dim.order_id +; + +-- Updated instance_nums in ib2bmedfact with order_ids +merge into i2b2medfact imf + using med_order_instance_map moim + on imf.instance_num=moim.deid_instance_num + when matched then + update set imf.instance_num=moim.deid_order_id +; \ No newline at end of file From 0a37f118e51adc59c551f4bbd61b72807abbe33f Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Dec 2016 13:11:15 -0600 Subject: [PATCH 202/507] Replaced non-functioning MERGE with UPDATE statement --- Oracle/observation_fact_meds.sql | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Oracle/observation_fact_meds.sql b/Oracle/observation_fact_meds.sql index b5b3708..4d755a3 100644 --- a/Oracle/observation_fact_meds.sql +++ b/Oracle/observation_fact_meds.sql @@ -118,9 +118,15 @@ join deid_id_order_id_mapping dim ; -- Updated instance_nums in ib2bmedfact with order_ids -merge into i2b2medfact imf - using med_order_instance_map moim - on imf.instance_num=moim.deid_instance_num - when matched then - update set imf.instance_num=moim.deid_order_id +update i2b2medfact imf +set imf.instance_num = ( + select moim.deid_order_id + from med_order_instance_map moim + where imf.instance_num=moim.deid_instance_num +) +where exists ( + select moim.deid_order_id + from med_order_instance_map moim + where imf.instance_num=moim.deid_instance_num +) ; \ No newline at end of file From 4c14e7e6601f1885678d420b8efc57b0be9e1304 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Dec 2016 13:33:48 -0600 Subject: [PATCH 203/507] Removed unnecessary WHERE EXISTS --- Oracle/observation_fact_meds.sql | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Oracle/observation_fact_meds.sql b/Oracle/observation_fact_meds.sql index 4d755a3..8799940 100644 --- a/Oracle/observation_fact_meds.sql +++ b/Oracle/observation_fact_meds.sql @@ -123,10 +123,5 @@ set imf.instance_num = ( select moim.deid_order_id from med_order_instance_map moim where imf.instance_num=moim.deid_instance_num -) -where exists ( - select moim.deid_order_id - from med_order_instance_map moim - where imf.instance_num=moim.deid_instance_num -) +) ; \ No newline at end of file From 96c18223a071b4c1c08f3bc47068771b70c890d0 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Dec 2016 14:03:57 -0600 Subject: [PATCH 204/507] Removed hard-coded refs --- Oracle/observation_fact_meds.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/observation_fact_meds.sql b/Oracle/observation_fact_meds.sql index 8799940..782ce6d 100644 --- a/Oracle/observation_fact_meds.sql +++ b/Oracle/observation_fact_meds.sql @@ -84,7 +84,7 @@ drop table med_order_instance_map; create table med_order_instance_map as with id_med_instance as ( select instance_num - from nightherondata.observation_fact@id + from "&&i2b2_id_data_schema".observation_fact"&&i2b2_id_data_link" where concept_cd like 'KUH|MEDICATION_ID:%' ), -- based on instance_num deid in i2b2_facts_deid.sql. @@ -123,5 +123,5 @@ set imf.instance_num = ( select moim.deid_order_id from med_order_instance_map moim where imf.instance_num=moim.deid_instance_num -) +) ; \ No newline at end of file From 663cce3cbed4081a1de1bb061e788b6a76954422 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Dec 2016 14:11:00 -0600 Subject: [PATCH 205/507] Substition var doesn't work with link --- Oracle/observation_fact_meds.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/observation_fact_meds.sql b/Oracle/observation_fact_meds.sql index 782ce6d..24b66b4 100644 --- a/Oracle/observation_fact_meds.sql +++ b/Oracle/observation_fact_meds.sql @@ -84,7 +84,7 @@ drop table med_order_instance_map; create table med_order_instance_map as with id_med_instance as ( select instance_num - from "&&i2b2_id_data_schema".observation_fact"&&i2b2_id_data_link" + from "&&i2b2_id_data_schema".observation_fact@id where concept_cd like 'KUH|MEDICATION_ID:%' ), -- based on instance_num deid in i2b2_facts_deid.sql. From 7cdc37fd63602b1bbf13b792429c813e41e4e027 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Dec 2016 14:14:15 -0600 Subject: [PATCH 206/507] Now able to handle attempt to drop non-existing table --- Oracle/observation_fact_meds.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Oracle/observation_fact_meds.sql b/Oracle/observation_fact_meds.sql index 24b66b4..0173607 100644 --- a/Oracle/observation_fact_meds.sql +++ b/Oracle/observation_fact_meds.sql @@ -79,7 +79,9 @@ create bitmap index MED_OBS_FACT_MOD_CODE_BI from (select 7 as num_bytes from dual); -- Generate deid instance_num order_id mapping +whenever sqlerror continue; drop table med_order_instance_map; +whenever sqlerror exit; create table med_order_instance_map as with id_med_instance as ( From f899e4426133e1c256f76e87d31a449b6b5a443b Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Dec 2016 16:06:34 -0600 Subject: [PATCH 207/507] Replaced expensive UPDATE with less expensive CREATE TABLE / DROP / RENAME --- Oracle/observation_fact_meds.sql | 38 +++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/Oracle/observation_fact_meds.sql b/Oracle/observation_fact_meds.sql index 0173607..66220ad 100644 --- a/Oracle/observation_fact_meds.sql +++ b/Oracle/observation_fact_meds.sql @@ -120,10 +120,36 @@ join deid_id_order_id_mapping dim ; -- Updated instance_nums in ib2bmedfact with order_ids -update i2b2medfact imf -set imf.instance_num = ( - select moim.deid_order_id - from med_order_instance_map moim - where imf.instance_num=moim.deid_instance_num -) +create table i2b2medfact2 as +select + imf.ENCOUNTER_NUM, + imf.PATIENT_NUM, + imf.CONCEPT_CD, + imf.PROVIDER_ID, + imf.START_DATE, + imf.MODIFIER_CD, + moim.deid_order_id, + imf.VALTYPE_CD, + imf.TVAL_CHAR, + imf.NVAL_NUM, + imf.VALUEFLAG_CD, + imf.QUANTITY_NUM, + imf.UNITS_CD, + imf.END_DATE, + imf.LOCATION_CD, + imf.OBSERVATION_BLOB, + imf.CONFIDENCE_NUM, + imf.UPDATE_DATE, + imf.DOWNLOAD_DATE, + imf.IMPORT_DATE, + imf.SOURCESYSTEM_CD, + imf.UPLOAD_ID, + imf.SUB_ENCOUNTER +from i2b2medfact imf +join med_order_instance_map moim + on imf.instance_num=moim.deid_instance_num +; + +drop table i2b2medfact; +rename table i2b2medfact2 to i2b2medfact; ; \ No newline at end of file From 9aa400f6d1a4cd438002f75172ce5746f2d00b3e Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Dec 2016 16:14:41 -0600 Subject: [PATCH 208/507] Updated to use correct table name instead of synonym --- Oracle/observation_fact_meds.sql | 62 +++++++++++++++++--------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/Oracle/observation_fact_meds.sql b/Oracle/observation_fact_meds.sql index 66220ad..3234656 100644 --- a/Oracle/observation_fact_meds.sql +++ b/Oracle/observation_fact_meds.sql @@ -120,36 +120,42 @@ join deid_id_order_id_mapping dim ; -- Updated instance_nums in ib2bmedfact with order_ids -create table i2b2medfact2 as +create table observation_fact_meds_2 as select - imf.ENCOUNTER_NUM, - imf.PATIENT_NUM, - imf.CONCEPT_CD, - imf.PROVIDER_ID, - imf.START_DATE, - imf.MODIFIER_CD, + ofm.ENCOUNTER_NUM, + ofm.PATIENT_NUM, + ofm.CONCEPT_CD, + ofm.PROVIDER_ID, + ofm.START_DATE, + ofm.MODIFIER_CD, moim.deid_order_id, - imf.VALTYPE_CD, - imf.TVAL_CHAR, - imf.NVAL_NUM, - imf.VALUEFLAG_CD, - imf.QUANTITY_NUM, - imf.UNITS_CD, - imf.END_DATE, - imf.LOCATION_CD, - imf.OBSERVATION_BLOB, - imf.CONFIDENCE_NUM, - imf.UPDATE_DATE, - imf.DOWNLOAD_DATE, - imf.IMPORT_DATE, - imf.SOURCESYSTEM_CD, - imf.UPLOAD_ID, - imf.SUB_ENCOUNTER -from i2b2medfact imf + ofm.VALTYPE_CD, + ofm.TVAL_CHAR, + ofm.NVAL_NUM, + ofm.VALUEFLAG_CD, + ofm.QUANTITY_NUM, + ofm.UNITS_CD, + ofm.END_DATE, + ofm.LOCATION_CD, + ofm.OBSERVATION_BLOB, + ofm.CONFIDENCE_NUM, + ofm.UPDATE_DATE, + ofm.DOWNLOAD_DATE, + ofm.IMPORT_DATE, + ofm.SOURCESYSTEM_CD, + ofm.UPLOAD_ID, + ofm.SUB_ENCOUNTER +from observation_fact_meds ofm join med_order_instance_map moim - on imf.instance_num=moim.deid_instance_num + on ofm.instance_num=moim.deid_instance_num ; -drop table i2b2medfact; -rename table i2b2medfact2 to i2b2medfact; -; \ No newline at end of file +whenever sqlerror continue; +drop table observation_fact_meds; +whenever sqlerror exit; + +rename table observation_fact_meds_2 to observation_fact_meds; + +whenever sqlerror continue; +drop table observation_fact_meds_2; +whenever sqlerror exit; \ No newline at end of file From 43eecccf4648ade5e6eeeb3b320ed8fd0d2fb472 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Dec 2016 16:18:52 -0600 Subject: [PATCH 209/507] Fixed table rename statement --- Oracle/observation_fact_meds.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/observation_fact_meds.sql b/Oracle/observation_fact_meds.sql index 3234656..ba6c711 100644 --- a/Oracle/observation_fact_meds.sql +++ b/Oracle/observation_fact_meds.sql @@ -154,7 +154,7 @@ whenever sqlerror continue; drop table observation_fact_meds; whenever sqlerror exit; -rename table observation_fact_meds_2 to observation_fact_meds; +alter table OBSERVATION_FACT_MEDS_2 rename to OBSERVATION_FACT_MEDS; whenever sqlerror continue; drop table observation_fact_meds_2; From bab94c87d503729ded061c4008ee105356877af3 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Dec 2016 16:21:10 -0600 Subject: [PATCH 210/507] Removed uneeded drop table after table rename --- Oracle/observation_fact_meds.sql | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Oracle/observation_fact_meds.sql b/Oracle/observation_fact_meds.sql index ba6c711..562777a 100644 --- a/Oracle/observation_fact_meds.sql +++ b/Oracle/observation_fact_meds.sql @@ -154,8 +154,4 @@ whenever sqlerror continue; drop table observation_fact_meds; whenever sqlerror exit; -alter table OBSERVATION_FACT_MEDS_2 rename to OBSERVATION_FACT_MEDS; - -whenever sqlerror continue; -drop table observation_fact_meds_2; -whenever sqlerror exit; \ No newline at end of file +alter table OBSERVATION_FACT_MEDS_2 rename to OBSERVATION_FACT_MEDS; \ No newline at end of file From 99423e7cdfa060a2a08c9cfd39cd056cfadb3d64 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 12 Dec 2016 08:37:28 -0600 Subject: [PATCH 211/507] Added needed instance_num alias to create table statement --- Oracle/observation_fact_meds.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/observation_fact_meds.sql b/Oracle/observation_fact_meds.sql index 562777a..e2e7f4e 100644 --- a/Oracle/observation_fact_meds.sql +++ b/Oracle/observation_fact_meds.sql @@ -128,7 +128,7 @@ select ofm.PROVIDER_ID, ofm.START_DATE, ofm.MODIFIER_CD, - moim.deid_order_id, + moim.deid_order_id instance_num, ofm.VALTYPE_CD, ofm.TVAL_CHAR, ofm.NVAL_NUM, From 1d227eb7c0b5fe6957f07a758ce5bc5a6a4bf622 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 12 Dec 2016 12:46:14 -0600 Subject: [PATCH 212/507] Update to correctly classify deceased patients with unknown death dates --- Oracle/PCORNetLoader_ora.sql | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index df58bba..c2d7e62 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1819,13 +1819,27 @@ create or replace procedure PCORNetDeath as begin insert into death( patid, death_date, death_date_impute, death_source, death_match_confidence) -select distinct pat.patient_num, pat.death_date, case -when vital_status_cd like 'X%' then 'B' -when vital_status_cd like 'M%' then 'D' -when upper(vital_status_cd) like 'Y%' then 'N' -else 'OT' -end, 'NI','NI' -from i2b2patient pat +select distinct pat.patient_num, pat.death_date, +case when vital_status_cd like 'X%' then 'B' + when vital_status_cd like 'M%' then 'D' + when upper(vital_status_cd) like 'Y%' then 'N' + else 'OT' + end death_date_impute, + 'NI' death_source, + 'NI' death_match_confidence +from ( + /* KUMC specific fix to address unknown death dates */ + select + ibp.patient_num, + case when ibf.concept_cd is not null then DATE '2100-12-31' + else ibp.death_date end death_date, + case when ibf.concept_cd is not null then 'OT' + else upper(ibp.vital_status_cd) end vital_status_cd + from i2b2patient ibp + left join i2b2fact ibf + on ibp.patient_num=ibf.patient_num + and ibf.CONCEPT_CD='DEM|VITAL:yu' +) pat where (pat.death_date is not null or vital_status_cd like 'Z%') and pat.patient_num in (select patid from demographic); end; From b380a9a1dfd6373493677bf6f994b22a8719587f Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 12 Dec 2016 14:40:45 -0600 Subject: [PATCH 213/507] Choose best RXCUI: SCD, SBC, GPCK, BPCK, or other (#4423) --- Oracle/pcornet_mapping.sql | 88 ++++++++++++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 13 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 9debde7..c07f7a3 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -363,26 +363,88 @@ and ht.m_exclusion_cd is null and ht.c_basecode is not null; -insert into "&&i2b2_meta_schema".PCORNET_MED -with -rxnorm_mapping as ( - /* TODO: Consider whether we want just one rxcui for a clarity medication? - Without picking just one this query results in duplicate c_fullnames and causes - errors in the webclient. + +/** medication_id_to_best_rxcui + +Spec for PRESCRIBING.RXNORM_CUI says: + + Where an RxNorm mapping exists for the source + medication, this field contains the RxNorm concept + identifier (CUI) at the highest possible specificity. + + If more than one option exists for mapping, the + following ordered strategy may be adopted: + 1)Semantic generic clinical drug + 2)Semantic Branded clinical drug + 3)Generic drug pack + 4)Branded drug pack + +*/ +create or replace view medication_id_to_best_rxcui as +with pcornet_spec as ( + select '1) Semantic generic clinical drug (SCD)' spec_order, 1 ix, 'SCD' tty from dual union all + select '2) Semantic Branded clinical drug (SBC)', 2, 'SBD' from dual union all + select '3) Generic drug pack (GPCK)', 3, 'GPCK' from dual union all + select '4) Branded drug pack (BPCK)', 4, 'BPCK' from dual +) +, cui_pref as ( -- Rx concepts joined with spec order + select rxcui, ix, spec_order + from rxnorm.rxnconso@id con + join pcornet_spec on pcornet_spec.tty = con.tty +) +, med_map_pref as ( -- HERON clarity_med_id_to_rxcui joined with spec order +/* TODO: Consider changing HERON paths to be RXCUIs or including the RXCUI column so that we don't have to reach back to the clarity_med_id_to_rxcui map. See also https://informatics.gpcnetwork.org/trac/Project/ticket/390. - */ - select min(rxcui) rxcui, clarity_med_id - from "&&i2b2_etl_schema". clarity_med_id_to_rxcui@id - group by clarity_med_id - ), +*/ + select distinct clarity_med_id medication_id, med_map.rxcui + , coalesce(spec_order, '9) HERON mapping misc.') spec_order + from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id med_map + join cui_pref on cui_pref.rxcui = med_map.rxcui +) +, med_map_best as ( + select * + from (select medication_id, rxcui, spec_order + , rank() over (partition by medication_id order by spec_order, rxcui) rnk + from med_map_pref) + where rnk = 1 +) +, all_med as ( + select cd. concept_cd, max(name_char) name_char + from blueherondata.concept_dimension cd + where cd.concept_cd like 'KUH|MEDICATION_ID:%' + group by cd.concept_cd +) +select all_med.*, med_map_best.* +from all_med +left join med_map_best on all_med.concept_cd = 'KUH|MEDICATION_ID:' || med_map_best.medication_id +; +; +/* +select * -- 26,647 rows with facts +from medication_id_to_rxcui mitr +join blueheronmetadata.counts_by_concept cbc on cbc.concept_cd = mitr.concept_cd + -- where rxcui is null -- 5,570 rows where rxcui is null +order by facts desc +; + +*/ + +/* test cases (TODO: formalize these) +select RXCUI from medication_id_to_rxcui where concept_cd = 'KUH|MEDICATION_ID:85051'; -- -> 1360019 +select concept_cd, name_char from medication_id_to_rxcui where rxcui = 892246; -- -> LEVOTHROID 100 MCG PO TAB etc. +*/ + + +insert into "&&i2b2_meta_schema".PCORNET_MED +with terms_rx as ( select - cm2rx.rxcui mapped_rxcui, ht.* + best.rxcui mapped_rxcui, ht.* from "&&i2b2_meta_schema"."&&terms_table" ht - left join rxnorm_mapping cm2rx on to_char(cm2rx.clarity_med_id) = replace(ht.c_basecode, 'KUH|MEDICATION_ID:', '') + left join medication_id_to_best_rxcui best on best.concept_cd = ht.c_basecode where c_fullname like '\i2b2\Medications%' and c_basecode not like 'NDC:%' -- We'll handle NDCs seperately below and ht.c_visualattributes not like '%H%' From 17879d3dc9f4cd9244492a20aca4acf8ed919590 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 12 Dec 2016 15:07:39 -0600 Subject: [PATCH 214/507] Minor comment and clean up --- Oracle/PCORNetLoader_ora.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index c2d7e62..a538c85 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1822,7 +1822,7 @@ insert into death( patid, death_date, death_date_impute, death_source, death_mat select distinct pat.patient_num, pat.death_date, case when vital_status_cd like 'X%' then 'B' when vital_status_cd like 'M%' then 'D' - when upper(vital_status_cd) like 'Y%' then 'N' + when vital_status_cd like 'Y%' then 'N' else 'OT' end death_date_impute, 'NI' death_source, @@ -1831,7 +1831,7 @@ from ( /* KUMC specific fix to address unknown death dates */ select ibp.patient_num, - case when ibf.concept_cd is not null then DATE '2100-12-31' + case when ibf.concept_cd is not null then DATE '2100-12-31' -- in accordance with the CDM v3 spec else ibp.death_date end death_date, case when ibf.concept_cd is not null then 'OT' else upper(ibp.vital_status_cd) end vital_status_cd From 37aa150ed603320486f45ebaa044e5e07902cadd Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 12 Dec 2016 15:08:58 -0600 Subject: [PATCH 215/507] explain (and cite source for ) PARTITION BY logic --- Oracle/pcornet_mapping.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index c07f7a3..b33a718 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -404,6 +404,11 @@ with pcornet_spec as ( join cui_pref on cui_pref.rxcui = med_map.rxcui ) , med_map_best as ( + -- Take the mapping with the best (i.e. min) spec_order, just like... + -- Taking the record with the max date + -- http://stackoverflow.com/a/8898142 + -- This may result in multiple rxcuis per medication_id, so while we're + -- at it, take the minimum rxcui among those with the best spec_order. select * from (select medication_id, rxcui, spec_order , rank() over (partition by medication_id order by spec_order, rxcui) rnk From a2db362d23b03ef5edd87c66d1445f4b347c84d2 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 12 Dec 2016 15:28:19 -0600 Subject: [PATCH 216/507] Oops! the 9) misc case needs a left join Thanks, Mike, for review. Elaborate "Eyeball it" queries and test-sketches a bit. --- Oracle/pcornet_mapping.sql | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index b33a718..b1f0690 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -401,7 +401,7 @@ with pcornet_spec as ( select distinct clarity_med_id medication_id, med_map.rxcui , coalesce(spec_order, '9) HERON mapping misc.') spec_order from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id med_map - join cui_pref on cui_pref.rxcui = med_map.rxcui + left join cui_pref on cui_pref.rxcui = med_map.rxcui ) , med_map_best as ( -- Take the mapping with the best (i.e. min) spec_order, just like... @@ -426,19 +426,33 @@ from all_med left join med_map_best on all_med.concept_cd = 'KUH|MEDICATION_ID:' || med_map_best.medication_id ; ; -/* -select * -- 26,647 rows with facts -from medication_id_to_rxcui mitr +/* Eyeball it: + +select * +from medication_id_to_best_rxcui mitr join blueheronmetadata.counts_by_concept cbc on cbc.concept_cd = mitr.concept_cd - -- where rxcui is null -- 5,570 rows where rxcui is null order by facts desc ; +How many of each spec_order? + +select count(*), spec_order from medication_id_to_best_rxcui +group by spec_order order by 1 desc; + +45773 1) Semantic generic clinical drug (SCD) + 5442 9) HERON mapping misc. + 3475 *null* lots of IVP. TODO: med mixes + 473 3) Generic drug pack (GPCK) + 97 2) Semantic Branded clinical drug (SBC) + 2 4) Branded drug pack (BPCK) */ /* test cases (TODO: formalize these) -select RXCUI from medication_id_to_rxcui where concept_cd = 'KUH|MEDICATION_ID:85051'; -- -> 1360019 -select concept_cd, name_char from medication_id_to_rxcui where rxcui = 892246; -- -> LEVOTHROID 100 MCG PO TAB etc. + +Don't map ENOXAPARIN 100 MG/ML SC SYRG to just any RXCUI: +select rxcui, spec_order from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:85051'; -- -> 1360019, 1) SCD + +select rxcui, spec_order from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:150171'; -- -> 892246, 1) SCD */ From 74f031e49fd0fe7a23f1b473a41e8e3b14adbe18 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 13 Dec 2016 13:38:21 -0600 Subject: [PATCH 217/507] when mapping to RXCUI, prefer GCN mappings to NDC - expand clarity_med_id_to_rxcui@id to its 3 constituent parts - Fix Enoxaparin tests (1360019 is the _wrong_ CUI that started us on this whole exercise.) - include clarity, RXNorm names to facilitate spot-checking - spot-check for picking more than one rxnorm row for a CUI --- Oracle/pcornet_mapping.sql | 75 +++++++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index b1f0690..8da0c0f 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -388,30 +388,46 @@ with pcornet_spec as ( select '4) Branded drug pack (BPCK)', 4, 'BPCK' from dual ) , cui_pref as ( -- Rx concepts joined with spec order - select rxcui, ix, spec_order + select rxcui, str rxnorm_str, ix, spec_order from rxnorm.rxnconso@id con join pcornet_spec on pcornet_spec.tty = con.tty ) -, med_map_pref as ( -- HERON clarity_med_id_to_rxcui joined with spec order /* TODO: Consider changing HERON paths to be RXCUIs or including the RXCUI column so that we don't have to reach back to the clarity_med_id_to_rxcui map. See also https://informatics.gpcnetwork.org/trac/Project/ticket/390. */ - select distinct clarity_med_id medication_id, med_map.rxcui +, clarity_med_id_to_rxcui as ( + select cmed.medication_id clarity_med_id, rxn.rxnorm_code rxcui, '1) Clarity' dose_pref + from clarity.rxnorm_codes@id rxn + join clarity.clarity_medication@id cmed on cmed.medication_id = rxn.medication_id + + union all + + select clarity_med_id, rxcui, '2) GCN' + from "&&i2b2_etl_schema".clarity_med_id_to_rxcui_gcn@id + + union all + + select clarity_med_id, rxcui, '3) NDC' + from "&&i2b2_etl_schema".clarity_med_id_to_rxcui_ndc@id +) +, med_map_pref as ( -- HERON clarity_med_id_to_rxcui joined with spec order + select distinct clarity_med_id medication_id + , med_map.rxcui, cui_pref.rxnorm_str, med_map.dose_pref , coalesce(spec_order, '9) HERON mapping misc.') spec_order - from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id med_map + from clarity_med_id_to_rxcui med_map left join cui_pref on cui_pref.rxcui = med_map.rxcui ) , med_map_best as ( - -- Take the mapping with the best (i.e. min) spec_order, just like... + -- Take the mapping with the best (i.e. min) spec_order and dose_pref, just like... -- Taking the record with the max date -- http://stackoverflow.com/a/8898142 -- This may result in multiple rxcuis per medication_id, so while we're -- at it, take the minimum rxcui among those with the best spec_order. select * - from (select medication_id, rxcui, spec_order - , rank() over (partition by medication_id order by spec_order, rxcui) rnk + from (select medication_id, rxcui, rxnorm_str, spec_order, dose_pref + , rank() over (partition by medication_id order by spec_order, dose_pref, rxcui) rnk from med_map_pref) where rnk = 1 ) @@ -433,26 +449,41 @@ from medication_id_to_best_rxcui mitr join blueheronmetadata.counts_by_concept cbc on cbc.concept_cd = mitr.concept_cd order by facts desc ; - -How many of each spec_order? - -select count(*), spec_order from medication_id_to_best_rxcui -group by spec_order order by 1 desc; - -45773 1) Semantic generic clinical drug (SCD) - 5442 9) HERON mapping misc. - 3475 *null* lots of IVP. TODO: med mixes - 473 3) Generic drug pack (GPCK) - 97 2) Semantic Branded clinical drug (SBC) - 2 4) Branded drug pack (BPCK) */ /* test cases (TODO: formalize these) -Don't map ENOXAPARIN 100 MG/ML SC SYRG to just any RXCUI: -select rxcui, spec_order from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:85051'; -- -> 1360019, 1) SCD +Picking out just one row per CUI from RXNORM can be tricky; did we goof? +select count(*), medication_id +from medication_id_to_best_rxcui +where medication_id is not null +group by medication_id having count(*) > 1; + +How many of each spec_order? -select rxcui, spec_order from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:150171'; -- -> 892246, 1) SCD +select count(*), spec_order, dose_pref from medication_id_to_best_rxcui +group by spec_order, dose_pref order by 1 desc; + +27825 1) Semantic generic clinical drug (SCD) 1) Clarity +17527 1) Semantic generic clinical drug (SCD) 2) GCN + 3475 *null* lots of IVP. TODO: med mixes + 2888 9) HERON mapping misc. 2) GCN + 2550 9) HERON mapping misc. 1) Clarity + 421 1) Semantic generic clinical drug (SCD) 3) NDC + 295 3) Generic drug pack (GPCK) 1) Clarity + 177 3) Generic drug pack (GPCK) 2) GCN + 97 2) Semantic Branded clinical drug (SBC) 3) NDC + 4 9) HERON mapping misc. 3) NDC + 2 4) Branded drug pack (BPCK) 3) NDC + 1 3) Generic drug pack (GPCK) 3) NDC + +Be sure RXCUI for 0.4 ML ENOXAPARIN 100 MG/ML SC SYRG includes the 0.4 ML dose info: +select concept_cd, name_char, rxcui, rxnorm_str, spec_order, dose_pref from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:85052'; -- -> 854235, 1) SCD + +select concept_cd, name_char, rxcui, rxnorm_str, spec_order, dose_pref from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:85051'; -- -> 854248, 1) SCD + +This one goes from "LEVOTHYROXINE PO" to "Levothyroxine Sodium 0.1 MG Oral Tablet"; is that right? +select concept_cd, name_char, rxcui, rxnorm_str, spec_order, dose_pref from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:150171'; -- -> 892246, 1) SCD */ From fb4755d6c6efd99935545d079b0013ef87b4cec8 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 13 Dec 2016 14:51:39 -0600 Subject: [PATCH 218/507] Calculate RX_DAYS_SUPPLY from RX_QUANTITY and RX_FREQUENCY where possible. --- Oracle/cdm_postproc.sql | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Oracle/cdm_postproc.sql b/Oracle/cdm_postproc.sql index 27790b2..419d6a5 100644 --- a/Oracle/cdm_postproc.sql +++ b/Oracle/cdm_postproc.sql @@ -25,3 +25,30 @@ The CDM wants height in inches and weight in pounds. */ update vital v set v.ht = v.ht / 2.54; update vital v set v.wt = v.wt / 16; +/* + * FIX: In order to populate RX_DAYS_SUPPLY for Cycle 2 we calculate it from + * RX_QUANTITY and RX_FREQUENCY. For Cycle 3 this should be removed as days + * supply values should make their way in from HERON. + */ +whenever sqlerror continue; +drop table pcori_freq_mapping; +whenever sqlerror exit; + +create table pcori_freq_mapping as ( + select '01' rx_frequency, 1 freq_daily from dual union all -- Every day + select '02' rx_frequency, 2 freq_daily from dual union all -- Two times a day + select '03' rx_frequency, 3 freq_daily from dual union all -- Three times a day + select '04' rx_frequency, 4 freq_daily from dual union all -- Four times a day + select '05' rx_frequency, 1 freq_daily from dual union all -- Every morning + select '06' rx_frequency, 1 freq_daily from dual union all -- Every afternoon + select '07' rx_frequency, 3 freq_daily from dual union all -- Before meals + select '08' rx_frequency, 3 freq_daily from dual -- After meals +); + +update prescribing pres +set pres.rx_days_supply = ( + select ceil(pres.rx_quantity / freq.freq_daily) + from pcori_freq_mapping freq +where pres.rx_frequency=freq.rx_frequency + and pres.rx_quantity is not null +); \ No newline at end of file From 8f473be4a6674bd3190a9cb8da50425d4a78564c Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 13 Dec 2016 15:07:39 -0600 Subject: [PATCH 219/507] fill in raw_rx_med_name from c_name and squirrel away c_basecode in raw_rxnorm_cui --- Oracle/PCORNetLoader_ora.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index a538c85..15464a7 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1641,13 +1641,14 @@ insert into prescribing ( ,RX_DAYS_SUPPLY -- modifier nval_num ,RX_FREQUENCY --modifier with basecode lookup ,RX_BASIS --modifier with basecode lookup --- ,RAW_RX_MED_NAME, --not filling these right now + ,RAW_RX_MED_NAME -- ,RAW_RX_FREQUENCY, --- ,RAW_RXNORM_CUI + ,RAW_RXNORM_CUI ) select distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH:MI'), m.start_date start_date, m.end_date, mo.pcori_cui ,quantity.nval_num quantity, refills.nval_num refills, supply.nval_num supply, substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) frequency, substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) basis + , substr(mo.c_name, 1, 50) raw_rx_med_name, substr(mo.c_basecode, 1, 50) raw_rxnorm_cui from i2b2medfact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode inner join encounter enc on enc.encounterid = m.encounter_Num -- TODO: This join adds several minutes to the load - must be debugged From ecca42224b6eb88d084fa853ac8fbcc1a33baf82 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 13 Dec 2016 16:33:17 -0600 Subject: [PATCH 220/507] avoid blank lines in medication_id_to_best_rxcui we seem to use this code in sqlplus without "blanklines" support --- Oracle/pcornet_mapping.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 8da0c0f..7efbc46 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -401,14 +401,10 @@ with pcornet_spec as ( select cmed.medication_id clarity_med_id, rxn.rxnorm_code rxcui, '1) Clarity' dose_pref from clarity.rxnorm_codes@id rxn join clarity.clarity_medication@id cmed on cmed.medication_id = rxn.medication_id - union all - select clarity_med_id, rxcui, '2) GCN' from "&&i2b2_etl_schema".clarity_med_id_to_rxcui_gcn@id - union all - select clarity_med_id, rxcui, '3) NDC' from "&&i2b2_etl_schema".clarity_med_id_to_rxcui_ndc@id ) From cb8b3ce107c76a2cccc37f407a8fe4933df855a1 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 13 Mar 2017 14:43:40 -0500 Subject: [PATCH 221/507] pcornet_mapping: include med curation (#4585) --- Oracle/pcornet_mapping.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 7efbc46..9635f53 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -401,12 +401,23 @@ with pcornet_spec as ( select cmed.medication_id clarity_med_id, rxn.rxnorm_code rxcui, '1) Clarity' dose_pref from clarity.rxnorm_codes@id rxn join clarity.clarity_medication@id cmed on cmed.medication_id = rxn.medication_id + union all + select clarity_med_id, rxcui, '2) GCN' from "&&i2b2_etl_schema".clarity_med_id_to_rxcui_gcn@id + union all + select clarity_med_id, rxcui, '3) NDC' from "&&i2b2_etl_schema".clarity_med_id_to_rxcui_ndc@id + + union all + + select clarity_medication_id, rxcui, '4) Manual Curation' --, con.tty, va_name, sdf_name + from heron_etl_1.med_map_manual_curation mmmc + join rxnorm.rxnconso@id con on con.rxaui = mmmc.sdf_rxaui + where con.tty not in ('FN') ) , med_map_pref as ( -- HERON clarity_med_id_to_rxcui joined with spec order select distinct clarity_med_id medication_id From 7f7320206dc591442e6acb743680b1ae69ce0a41 Mon Sep 17 00:00:00 2001 From: Matt Hoag Date: Thu, 16 Mar 2017 09:45:18 -0500 Subject: [PATCH 222/507] =?UTF-8?q?Added=20additional=20PCORI->HERON=20enc?= =?UTF-8?q?ounter=20type=20mappings=20(particularly=20`OA`)=20=20=E2=80=94?= =?UTF-8?q?=20generated=20from:=20=20=20=20select=20replace(c=5Ffullname,'?= =?UTF-8?q?\i2b2\Visit=20Details','\PCORI\ENCOUNTER')||','||c=5Ffullname?= =?UTF-8?q?=20as=20mapping=20=20=20=20from=20(select=20c=5Ffullname=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20from=20heron=5Fterms=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20where=20c=5Ffullname=20like=20'\i2b2\Visit=20Detail?= =?UTF-8?q?s\ENC=5FTYPE\%'=20and=20c=5Fhlevel=20=3D=203=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20order=20by=20c=5Fhlevel,=20c=5Ffullname);?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Oracle/pcornet_mapping.csv | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index d380049..fee21ee 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -26,10 +26,14 @@ pcori_path,local_path \PCORI\DEMOGRAPHIC\HISPANIC\Y\,\i2b2\Demographics\Ethnicity\Hispanic or Latino\ \PCORI\DEMOGRAPHIC\HISPANIC\Y\,"\i2b2\Demographics\Ethnicity\Hispanic, Latino or Spanish Origin\" \PCORI\ENCOUNTER\ENC_TYPE\AV\,\i2b2\Visit Details\ENC_TYPE\AV\ -\PCORI\ENCOUNTER\ENC_TYPE\AV\,\i2b2\Visit Details\ENC_TYPE\AV\ \PCORI\ENCOUNTER\ENC_TYPE\ED\,\i2b2\Visit Details\ENC_TYPE\ED\ +\PCORI\ENCOUNTER\ENC_TYPE\EI\,\i2b2\Visit Details\ENC_TYPE\EI\ \PCORI\ENCOUNTER\ENC_TYPE\IP\,\i2b2\Visit Details\ENC_TYPE\IP\ +\PCORI\ENCOUNTER\ENC_TYPE\IS\,\i2b2\Visit Details\ENC_TYPE\IS\ +\PCORI\ENCOUNTER\ENC_TYPE\NI\,\i2b2\Visit Details\ENC_TYPE\NI\ +\PCORI\ENCOUNTER\ENC_TYPE\OA\,\i2b2\Visit Details\ENC_TYPE\OA\ \PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\ENC_TYPE\OT\ +\PCORI\ENCOUNTER\ENC_TYPE\UN\,\i2b2\Visit Details\ENC_TYPE\UN\ \PCORI\DEMOGRAPHIC\SEX\F\,\i2b2\Demographics\Gender\Female\ \PCORI\DEMOGRAPHIC\SEX\M\,\i2b2\Demographics\Gender\Male\ \PCORI\DEMOGRAPHIC\SEX\NI\,\i2b2\Demographics\Gender\Unknown\Unknown-@\ From 095ea90aea10d32c54c5e7ff53468012794bff68 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 30 Mar 2017 14:36:21 -0500 Subject: [PATCH 223/507] Updated modifier mappings to include dispensing quantity and days supply. --- Oracle/pcornet_mapping.csv | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index fee21ee..dbae26c 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -1,5 +1,7 @@ pcori_path,local_path -\PCORI_MOD\RX_BASIS\DI\,\Medication\Surescripts\ +\PCORI_MOD\RX_BASIS\DI\,\Medication\Surescripts\Status\ +\PCORI_MOD\PCORI_MOD\RX_QUANTITY\,\Medication\Surescripts\Amount\ +\PCORI_MOD\PCORI_MOD\RX_DAYS_SUPPLY\,\Medication\Surescripts\Days Supply\ \PCORI_MOD\RX_BASIS\PR\02\,\Medication\Inpatient\ \PCORI_MOD\RX_BASIS\PR\OT\,\Medication\Other Orders\ \PCORI_MOD\RX_BASIS\PR\01\,\Medication\Outpatient\ From d18c94f8a73c01b73d110303d82bf73f6d860722 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 30 Mar 2017 15:26:54 -0500 Subject: [PATCH 224/507] Update dispensing transform to include quantity and days supply. --- Oracle/PCORNetLoader_ora.sql | 62 ++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 15464a7..3e0dc36 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1748,27 +1748,57 @@ PMN_EXECUATESQL(sqltext); insert into dispensing ( PATID - ,PRESCRIBINGID + ,PRESCRIBINGID ,DISPENSE_DATE -- using start_date from i2b2 - ,NDC --using pcornet_med pcori_ndc - new column! - ,DISPENSE_SUP ---- modifier nval_num - ,DISPENSE_AMT -- modifier nval_num + ,NDC --using pcornet_med pcori_ndc - new column! + ,DISPENSE_SUP ---- modifier nval_num + ,DISPENSE_AMT -- modifier nval_num -- ,RAW_NDC ) /* Below is the Cycle 2 fix for populating the DISPENSING table */ -select distinct - ibf.patient_num patid, +with disp_status as ( + select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd + from i2b2fact ibf + join BLUEHERONMETADATA.pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode + where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' + and length(substr(ibf.concept_cd, 5)) < 12 +) +, disp_quantity as ( + select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num + from i2b2fact ibf + join BLUEHERONMETADATA.pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode + where pnm.c_fullname like '\PCORI_MOD\PCORI_MOD\RX_QUANTITY\%' +) +, disp_supply as ( + select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num + from i2b2fact ibf + join BLUEHERONMETADATA.pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode + where pnm.c_fullname like '\PCORI_MOD\PCORI_MOD\RX_DAYS_SUPPLY\%' +) +select + st.patient_num patid, null prescribingid, - ibf.start_date dispense_date, - substr(ibf.concept_cd, 5) ndc, - null dispense_sup, - null dispense_amt -from i2b2fact ibf -join BLUEHERONMETADATA.pcornet_med pnm - on ibf.modifier_cd=pnm.c_basecode -where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' - and length(substr(ibf.concept_cd, 5)) < 12 -; + st.start_date dispense_date, + substr(st.concept_cd, 5) ndc, + qt.nval_num dispense_sup, + qt.nval_num dispense_amt +from disp_status st +left outer join disp_quantity qt + on st.patient_num=qt.patient_num + and st.encounter_num=qt.encounter_num + and st.concept_cd=qt.concept_cd + and st.instance_num=qt.instance_num + and st.start_date=qt.start_date +left outer join disp_supply ds + on st.patient_num=ds.patient_num + and st.encounter_num=ds.encounter_num + and st.concept_cd=ds.concept_cd + and st.instance_num=ds.instance_num + and st.start_date=ds.start_date +; /* NOTE: Once DISPENSING related encounters have made it into the CDM (via the visit dimension) then the original SCILHS code below should work. From dff5e5eec3674aeab0eaa979da626561a48ecf7a Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 3 Apr 2017 10:34:44 -0500 Subject: [PATCH 225/507] Removed RX_DAYS_SUPPLY post proc fix. --- Oracle/cdm_postproc.sql | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/Oracle/cdm_postproc.sql b/Oracle/cdm_postproc.sql index 419d6a5..57c4a9d 100644 --- a/Oracle/cdm_postproc.sql +++ b/Oracle/cdm_postproc.sql @@ -23,32 +23,4 @@ when matched then update set p.rx_providerid = e.providerid; /* Currently in HERON, we have hight in cm and weight in oz (from visit vitals). The CDM wants height in inches and weight in pounds. */ update vital v set v.ht = v.ht / 2.54; -update vital v set v.wt = v.wt / 16; - -/* - * FIX: In order to populate RX_DAYS_SUPPLY for Cycle 2 we calculate it from - * RX_QUANTITY and RX_FREQUENCY. For Cycle 3 this should be removed as days - * supply values should make their way in from HERON. - */ -whenever sqlerror continue; -drop table pcori_freq_mapping; -whenever sqlerror exit; - -create table pcori_freq_mapping as ( - select '01' rx_frequency, 1 freq_daily from dual union all -- Every day - select '02' rx_frequency, 2 freq_daily from dual union all -- Two times a day - select '03' rx_frequency, 3 freq_daily from dual union all -- Three times a day - select '04' rx_frequency, 4 freq_daily from dual union all -- Four times a day - select '05' rx_frequency, 1 freq_daily from dual union all -- Every morning - select '06' rx_frequency, 1 freq_daily from dual union all -- Every afternoon - select '07' rx_frequency, 3 freq_daily from dual union all -- Before meals - select '08' rx_frequency, 3 freq_daily from dual -- After meals -); - -update prescribing pres -set pres.rx_days_supply = ( - select ceil(pres.rx_quantity / freq.freq_daily) - from pcori_freq_mapping freq -where pres.rx_frequency=freq.rx_frequency - and pres.rx_quantity is not null -); \ No newline at end of file +update vital v set v.wt = v.wt / 16; \ No newline at end of file From af38b2f4602cd145916622bc7d9db2dd469361c3 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 3 Apr 2017 14:25:16 -0500 Subject: [PATCH 226/507] A few bug fixes, comments, and todos --- Oracle/PCORNetLoader_ora.sql | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 3e0dc36..548c931 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1723,7 +1723,7 @@ whenever sqlerror exit; create or replace procedure PCORNetDispensing as sqltext varchar2(4000); begin - +/* PMN_DROPSQL('drop index dispensing_patid'); PMN_DROPSQL('DROP TABLE supply'); @@ -1743,13 +1743,14 @@ sqltext := 'create table amount as '|| ' on amount.modifier_cd = amountcode.c_basecode '|| ' and amountcode.c_fullname like ''\PCORI_MOD\RX_QUANTITY\'') '; PMN_EXECUATESQL(sqltext); - --- insert data with outer joins to ensure all records are included even if some data elements are missing +*/ + +/* NOTE: New transformation developed by KUMC */ insert into dispensing ( PATID ,PRESCRIBINGID - ,DISPENSE_DATE -- using start_date from i2b2 + ,DISPENSE_DATE -- using start_date from i2b2 ,NDC --using pcornet_med pcori_ndc - new column! ,DISPENSE_SUP ---- modifier nval_num ,DISPENSE_AMT -- modifier nval_num @@ -1758,22 +1759,22 @@ insert into dispensing ( /* Below is the Cycle 2 fix for populating the DISPENSING table */ with disp_status as ( select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd - from i2b2fact ibf + from i2b2medfact ibf join BLUEHERONMETADATA.pcornet_med pnm on ibf.modifier_cd=pnm.c_basecode where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' - and length(substr(ibf.concept_cd, 5)) < 12 + and length(substr(ibf.concept_cd, 5)) < 12 -- TODO: Generalize this for other sites. ) , disp_quantity as ( select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num - from i2b2fact ibf + from i2b2medfact ibf join BLUEHERONMETADATA.pcornet_med pnm on ibf.modifier_cd=pnm.c_basecode where pnm.c_fullname like '\PCORI_MOD\PCORI_MOD\RX_QUANTITY\%' ) , disp_supply as ( select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num - from i2b2fact ibf + from i2b2medfact ibf join BLUEHERONMETADATA.pcornet_med pnm on ibf.modifier_cd=pnm.c_basecode where pnm.c_fullname like '\PCORI_MOD\PCORI_MOD\RX_DAYS_SUPPLY\%' @@ -1782,8 +1783,8 @@ select st.patient_num patid, null prescribingid, st.start_date dispense_date, - substr(st.concept_cd, 5) ndc, - qt.nval_num dispense_sup, + substr(st.concept_cd, 5) ndc, -- TODO: Generalize this for other sites. + ds.nval_num dispense_sup, qt.nval_num dispense_amt from disp_status st left outer join disp_quantity qt @@ -1800,8 +1801,9 @@ left outer join disp_supply ds and st.start_date=ds.start_date ; -/* NOTE: Once DISPENSING related encounters have made it into the CDM (via the visit - dimension) then the original SCILHS code below should work. +/* NOTE: The original SCILHS transformation is below. + +-- insert data with outer joins to ensure all records are included even if some data elements are missing select m.patient_num, null,m.start_date, NVL(mo.pcori_ndc,'NA') ,max(supply.nval_num) sup, max(amount.nval_num) amt From d6619eb47c495a016f6ad1bba763c2184851808b Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 3 Apr 2017 15:12:30 -0500 Subject: [PATCH 227/507] Fixed typo in mapping --- Oracle/pcornet_mapping.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index dbae26c..2667784 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -1,7 +1,7 @@ pcori_path,local_path \PCORI_MOD\RX_BASIS\DI\,\Medication\Surescripts\Status\ -\PCORI_MOD\PCORI_MOD\RX_QUANTITY\,\Medication\Surescripts\Amount\ -\PCORI_MOD\PCORI_MOD\RX_DAYS_SUPPLY\,\Medication\Surescripts\Days Supply\ +\PCORI_MOD\RX_QUANTITY\,\Medication\Surescripts\Amount\ +\PCORI_MOD\RX_DAYS_SUPPLY\,\Medication\Surescripts\Days Supply\ \PCORI_MOD\RX_BASIS\PR\02\,\Medication\Inpatient\ \PCORI_MOD\RX_BASIS\PR\OT\,\Medication\Other Orders\ \PCORI_MOD\RX_BASIS\PR\01\,\Medication\Outpatient\ From 378a4094d0cb72d25d85db06e87118c45f452cfc Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 3 Apr 2017 15:14:06 -0500 Subject: [PATCH 228/507] Doesn't seem to be working against i2b2medfact --- Oracle/PCORNetLoader_ora.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 548c931..ca254ef 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1759,7 +1759,7 @@ insert into dispensing ( /* Below is the Cycle 2 fix for populating the DISPENSING table */ with disp_status as ( select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd - from i2b2medfact ibf + from i2b2fact ibf join BLUEHERONMETADATA.pcornet_med pnm on ibf.modifier_cd=pnm.c_basecode where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' @@ -1767,14 +1767,14 @@ with disp_status as ( ) , disp_quantity as ( select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num - from i2b2medfact ibf + from i2b2fact ibf join BLUEHERONMETADATA.pcornet_med pnm on ibf.modifier_cd=pnm.c_basecode where pnm.c_fullname like '\PCORI_MOD\PCORI_MOD\RX_QUANTITY\%' ) , disp_supply as ( select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num - from i2b2medfact ibf + from i2b2fact ibf join BLUEHERONMETADATA.pcornet_med pnm on ibf.modifier_cd=pnm.c_basecode where pnm.c_fullname like '\PCORI_MOD\PCORI_MOD\RX_DAYS_SUPPLY\%' From 50cad13ff964d2a87e035e934f6431710e1ae86d Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 4 Apr 2017 08:22:58 -0500 Subject: [PATCH 229/507] Fixed modifier path typos in PCORNetDispensing procedure --- Oracle/PCORNetLoader_ora.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index ca254ef..0749e2f 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1770,14 +1770,14 @@ with disp_status as ( from i2b2fact ibf join BLUEHERONMETADATA.pcornet_med pnm on ibf.modifier_cd=pnm.c_basecode - where pnm.c_fullname like '\PCORI_MOD\PCORI_MOD\RX_QUANTITY\%' + where pnm.c_fullname like '\PCORI_MOD\RX_QUANTITY\%' ) , disp_supply as ( select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num from i2b2fact ibf join BLUEHERONMETADATA.pcornet_med pnm on ibf.modifier_cd=pnm.c_basecode - where pnm.c_fullname like '\PCORI_MOD\PCORI_MOD\RX_DAYS_SUPPLY\%' + where pnm.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\%' ) select st.patient_num patid, From 9423a3f966cafeddf55ba9c4128cd44bcac2a59d Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Fri, 7 Apr 2017 09:14:15 -0500 Subject: [PATCH 230/507] Update schema references in and add documentation to PCORNetDispensing procedure --- Oracle/PCORNetLoader_ora.sql | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 0749e2f..6c7f760 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1760,22 +1760,25 @@ insert into dispensing ( with disp_status as ( select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd from i2b2fact ibf - join BLUEHERONMETADATA.pcornet_med pnm + join "&&i2b2_meta_schema".pcornet_med pnm on ibf.modifier_cd=pnm.c_basecode where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' - and length(substr(ibf.concept_cd, 5)) < 12 -- TODO: Generalize this for other sites. + /* TODO: Generalize for other sites. The substr() strips 'NDC:' from the + front the concept_cd and the '< 12' makes sure only 11 digit codes + are included. */ + and length(substr(ibf.concept_cd, 5)) < 12 ) , disp_quantity as ( select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num from i2b2fact ibf - join BLUEHERONMETADATA.pcornet_med pnm + join "&&i2b2_meta_schema".pcornet_med pnm on ibf.modifier_cd=pnm.c_basecode where pnm.c_fullname like '\PCORI_MOD\RX_QUANTITY\%' ) , disp_supply as ( select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num from i2b2fact ibf - join BLUEHERONMETADATA.pcornet_med pnm + join "&&i2b2_meta_schema".pcornet_med pnm on ibf.modifier_cd=pnm.c_basecode where pnm.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\%' ) From 5a2fbbe52d4a09830a54d539a143e9c9093e6165 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Fri, 7 Apr 2017 13:09:25 -0500 Subject: [PATCH 231/507] Updated SQL for readability --- Oracle/PCORNetLoader_ora.sql | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 6c7f760..1af39be 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1763,10 +1763,9 @@ with disp_status as ( join "&&i2b2_meta_schema".pcornet_med pnm on ibf.modifier_cd=pnm.c_basecode where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' - /* TODO: Generalize for other sites. The substr() strips 'NDC:' from the - front the concept_cd and the '< 12' makes sure only 11 digit codes - are included. */ - and length(substr(ibf.concept_cd, 5)) < 12 + /* TODO: Generalize for other sites. The '< 12' makes sure only 11 digit + codes are included. */ + and length(replace(concept_cd, 'NDC:', '')) < 12 ) , disp_quantity as ( select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num @@ -1786,7 +1785,7 @@ select st.patient_num patid, null prescribingid, st.start_date dispense_date, - substr(st.concept_cd, 5) ndc, -- TODO: Generalize this for other sites. + replace(concept_cd, 'NDC:', '') ndc, -- TODO: Generalize this for other sites. ds.nval_num dispense_sup, qt.nval_num dispense_amt from disp_status st From 7505a63687cf57d04b5d51020c9da2a76a0be2b4 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 11 Apr 2017 10:12:12 -0500 Subject: [PATCH 232/507] Initial pass at including all LOINC labs into CDM --- Oracle/PCORNetLoader_ora.sql | 4 +- Oracle/lab_loinc_mapping.csv | 156 +++++++++++++++++++++++++++++++++++ Oracle/pcornet_mapping.sql | 138 +++++++++++++++++++++---------- Oracle/run-i2p-transform.sh | 1 + 4 files changed, 254 insertions(+), 45 deletions(-) create mode 100644 Oracle/lab_loinc_mapping.csv diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 15464a7..99101f9 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1434,7 +1434,7 @@ inner join encounter enc on enc.patid = m.patient_num and enc.encounterid = m.en inner join pcornet_lab lab on lab.c_basecode = M.concept_cd and lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' inner JOIN pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname -inner join pmn_labnormal norm on ont_parent.c_basecode=norm.LAB_NAME +left outer join pmn_labnormal norm on ont_parent.c_basecode=norm.LAB_NAME LEFT OUTER JOIN priority p @@ -1453,7 +1453,7 @@ and M.concept_cd=l.concept_Cd and M.start_date=l.start_Date WHERE m.ValType_Cd in ('N','T') -and ont_parent.C_BASECODE LIKE 'LAB_NAME%' -- Exclude non-pcori labs +--and ont_parent.C_BASECODE LIKE 'LAB_NAME%' -- Exclude non-pcori labs and m.MODIFIER_CD='@'; execute immediate 'create index lab_result_cm_patid on lab_result_cm (PATID)'; diff --git a/Oracle/lab_loinc_mapping.csv b/Oracle/lab_loinc_mapping.csv new file mode 100644 index 0000000..9338321 --- /dev/null +++ b/Oracle/lab_loinc_mapping.csv @@ -0,0 +1,156 @@ +"LOINC_CODE","LAB_NAME","PCORI_SPECIMEN_SOURCE" +"13457-7","LDL","" +"18261-8","LDL","" +"2089-1","LDL","" +"47213-4","LDL","SERUM, PLASMA, or SR_PLS" +"47213-4","LDL","SERUM, PLASMA, or SR_PLS" +"47213-4","LDL","SERUM, PLASMA, or SR_PLS" +"49132-4","LDL","" +"17855-8","A1C","" +"17856-6","A1C","BLOOD" +"4548-4","A1C","BLOOD" +"59261-8","A1C","" +"62388-4","A1C","" +"2157-6","CK","SERUM, PLASMA, or SR_PLS" +"2157-6","CK","SERUM, PLASMA, or SR_PLS" +"2157-6","CK","SERUM, PLASMA, or SR_PLS" +"2157-6","CK","SERUM, PLASMA, or SR_PLS" +"50756-6","CK","BLOOD" +"50756-6","CK","BLOOD" +"50756-6","CK","BLOOD" +"50756-6","CK","BLOOD" +"13969-1","CK_MB","SERUM, PLASMA, or SR_PLS" +"13969-1","CK_MB","SERUM, PLASMA, or SR_PLS" +"13969-1","CK_MB","SERUM, PLASMA, or SR_PLS" +"13969-1","CK_MB","SERUM, PLASMA, or SR_PLS" +"2154-3","CK_MB","SERUM, PLASMA, or SR_PLS" +"2154-3","CK_MB","SERUM, PLASMA, or SR_PLS" +"2154-3","CK_MB","SERUM, PLASMA, or SR_PLS" +"2154-3","CK_MB","SERUM, PLASMA, or SR_PLS" +"32673-6","CK_MB","SERUM, PLASMA, or SR_PLS" +"32673-6","CK_MB","SERUM, PLASMA, or SR_PLS" +"32673-6","CK_MB","SERUM, PLASMA, or SR_PLS" +"32673-6","CK_MB","SERUM, PLASMA, or SR_PLS" +"49551-5","CK_MB","BLOOD" +"49551-5","CK_MB","BLOOD" +"49551-5","CK_MB","BLOOD" +"49551-5","CK_MB","BLOOD" +"12187-1","CK_MBI","SERUM, PLASMA, or SR_PLS" +"12187-1","CK_MBI","SERUM, PLASMA, or SR_PLS" +"12187-1","CK_MBI","SERUM, PLASMA, or SR_PLS" +"12187-1","CK_MBI","SERUM, PLASMA, or SR_PLS" +"12189-7","CK_MBI","SERUM, PLASMA, or SR_PLS" +"12189-7","CK_MBI","SERUM, PLASMA, or SR_PLS" +"12189-7","CK_MBI","SERUM, PLASMA, or SR_PLS" +"12189-7","CK_MBI","SERUM, PLASMA, or SR_PLS" +"20569-0","CK_MBI","SERUM, PLASMA, or SR_PLS" +"20569-0","CK_MBI","SERUM, PLASMA, or SR_PLS" +"20569-0","CK_MBI","SERUM, PLASMA, or SR_PLS" +"20569-0","CK_MBI","SERUM, PLASMA, or SR_PLS" +"49136-5","CK_MBI","SERUM, PLASMA, or SR_PLS" +"49136-5","CK_MBI","SERUM, PLASMA, or SR_PLS" +"49136-5","CK_MBI","SERUM, PLASMA, or SR_PLS" +"49136-5","CK_MBI","SERUM, PLASMA, or SR_PLS" +"2160-0","CREATININE","SERUM, PLASMA, or SR_PLS" +"38483-4","CREATININE","BLOOD" +"14682-9","CREATININE","SERUM, PLASMA, or SR_PLS" +"14682-9","CREATININE","SERUM, PLASMA, or SR_PLS" +"14682-9","CREATININE","SERUM, PLASMA, or SR_PLS" +"14682-9","CREATININE","SERUM, PLASMA, or SR_PLS" +"21232-4","CREATININE","BLOOD" +"21232-4","CREATININE","BLOOD" +"21232-4","CREATININE","BLOOD" +"21232-4","CREATININE","BLOOD" +"35203-9","CREATININE","SERUM, PLASMA, or SR_PLS" +"35203-9","CREATININE","SERUM, PLASMA, or SR_PLS" +"35203-9","CREATININE","SERUM, PLASMA, or SR_PLS" +"35203-9","CREATININE","SERUM, PLASMA, or SR_PLS" +"44784-7","CREATININE","SERUM, PLASMA, or SR_PLS" +"44784-7","CREATININE","SERUM, PLASMA, or SR_PLS" +"44784-7","CREATININE","SERUM, PLASMA, or SR_PLS" +"44784-7","CREATININE","SERUM, PLASMA, or SR_PLS" +"59826-8","CREATININE","BLOOD" +"59826-8","CREATININE","BLOOD" +"59826-8","CREATININE","BLOOD" +"59826-8","CREATININE","BLOOD" +"14775-1","HGB","BLOOD" +"14775-1","HGB","BLOOD" +"14775-1","HGB","BLOOD" +"14775-1","HGB","BLOOD" +"20509-6","HGB","BLOOD" +"20509-6","HGB","BLOOD" +"20509-6","HGB","BLOOD" +"20509-6","HGB","BLOOD" +"30313-1","HGB","BLOOD" +"30313-1","HGB","BLOOD" +"30313-1","HGB","BLOOD" +"30313-1","HGB","BLOOD" +"30350-3","HGB","BLOOD" +"30350-3","HGB","BLOOD" +"30350-3","HGB","BLOOD" +"30350-3","HGB","BLOOD" +"30351-1","HGB","BLOOD" +"30351-1","HGB","BLOOD" +"30351-1","HGB","BLOOD" +"30351-1","HGB","BLOOD" +"30352-9","HGB","BLOOD" +"30352-9","HGB","BLOOD" +"30352-9","HGB","BLOOD" +"30352-9","HGB","BLOOD" +"55782-7","HGB","BLOOD" +"55782-7","HGB","BLOOD" +"55782-7","HGB","BLOOD" +"55782-7","HGB","BLOOD" +"59260-0","HGB","BLOOD" +"59260-0","HGB","BLOOD" +"59260-0","HGB","BLOOD" +"59260-0","HGB","BLOOD" +"718-7","HGB","BLOOD" +"34714-6","INR","BLOOD" +"34714-6","INR","BLOOD" +"34714-6","INR","BLOOD" +"34714-6","INR","BLOOD" +"46418-0","INR","BLOOD" +"46418-0","INR","BLOOD" +"46418-0","INR","BLOOD" +"46418-0","INR","BLOOD" +"6301-6","INR","PPP" +"6301-6","INR","PPP" +"6301-6","INR","PPP" +"6301-6","INR","PPP" +"10839-9","TROP_I","SERUM, PLASMA, or SR_PLS" +"10839-9","TROP_I","SERUM, PLASMA, or SR_PLS" +"10839-9","TROP_I","SERUM, PLASMA, or SR_PLS" +"10839-9","TROP_I","SERUM, PLASMA, or SR_PLS" +"16255-2","TROP_I","SERUM, PLASMA, or SR_PLS" +"16255-2","TROP_I","SERUM, PLASMA, or SR_PLS" +"16255-2","TROP_I","SERUM, PLASMA, or SR_PLS" +"16255-2","TROP_I","SERUM, PLASMA, or SR_PLS" +"42757-5","TROP_I","BLOOD" +"42757-5","TROP_I","BLOOD" +"42757-5","TROP_I","BLOOD" +"42757-5","TROP_I","BLOOD" +"49563-0","TROP_I","SERUM, PLASMA, or SR_PLS" +"49563-0","TROP_I","SERUM, PLASMA, or SR_PLS" +"49563-0","TROP_I","SERUM, PLASMA, or SR_PLS" +"49563-0","TROP_I","SERUM, PLASMA, or SR_PLS" +"33204-9","TROP_T_QL","SERUM, PLASMA, or SR_PLS" +"33204-9","TROP_T_QL","SERUM, PLASMA, or SR_PLS" +"33204-9","TROP_T_QL","SERUM, PLASMA, or SR_PLS" +"33204-9","TROP_T_QL","SERUM, PLASMA, or SR_PLS" +"48426-1","TROP_T_QL","BLOOD" +"48426-1","TROP_T_QL","BLOOD" +"48426-1","TROP_T_QL","BLOOD" +"48426-1","TROP_T_QL","BLOOD" +"48425-3","TROP_T_QN","BLOOD" +"48425-3","TROP_T_QN","BLOOD" +"48425-3","TROP_T_QN","BLOOD" +"48425-3","TROP_T_QN","BLOOD" +"6597-9","TROP_T_QN","BLOOD" +"6597-9","TROP_T_QN","BLOOD" +"6597-9","TROP_T_QN","BLOOD" +"6597-9","TROP_T_QN","BLOOD" +"6598-7","TROP_T_QN","SERUM, PLASMA, or SR_PLS" +"6598-7","TROP_T_QN","SERUM, PLASMA, or SR_PLS" +"6598-7","TROP_T_QN","SERUM, PLASMA, or SR_PLS" +"6598-7","TROP_T_QN","SERUM, PLASMA, or SR_PLS" diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 7efbc46..5a0ecfa 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -581,53 +581,105 @@ join "&&i2b2_meta_schema".pcornet_med on med_mod_mapping.PCORI_PATH = pcornet_me commit; -/* Add relevent nodes from local i2b2 lab hierarchy to PCORNet Labs hierarchy. +/* Replace SCILHS PCORNet Labs hierarchy with our local LOINC hierarchy with + adjustment to make it "SCILHS like". */ +/* Remove existing SCIHLS Labs hierarchy */ +truncate table "&&i2b2_meta_schema".pcornet_lab; + +/* Insert LOINC codes from local hierarchy, setting c_basecode to appropriate + LAB_NAME for common PCORNet labs. */ +insert into "&&i2b2_meta_schema".pcornet_lab +select + lc.C_HLEVEL, + replace(lc.C_FULLNAME, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_fullname, + lc.C_NAME, + lc.C_SYNONYM_CD, + lc.C_VISUALATTRIBUTES, + lc.C_TOTALNUM, + case + when llm.lab_name is not null then 'LAB_NAME:'|| llm.lab_name + else lc.c_basecode + end c_basecose, + lc.C_METADATAXML, + lc.C_FACTTABLECOLUMN, + lc.C_TABLENAME, + lc.C_COLUMNNAME, + lc.C_COLUMNDATATYPE, + lc.C_OPERATOR, + lc.C_DIMCODE, + lc.C_COMMENT, + lc.C_TOOLTIP, + lc.M_APPLIED_PATH, + lc.UPDATE_DATE, + lc.DOWNLOAD_DATE, + lc.IMPORT_DATE, + lc.SOURCESYSTEM_CD, + lc.VALUETYPE_CD, + lc.M_EXCLUSION_CD, + lc.C_PATH, + lc.C_SYMBOL, + llm.pcori_specimen_source, + replace(lc.c_basecode, 'LOINC:', '') pcori_basecode +from "&&i2b2_meta_schema"."&&terms_table" lc +left outer join lab_loinc_mapping llm + on lc.c_basecode = ('LOINC:' || llm.loinc_code) +where lc.c_basecode like 'LOINC:%' +; + +/* Insert child KUH|COMPONENT_ID nodes, setting pcori_basecode, and c_path to + that of it's parent. */ insert into "&&i2b2_meta_schema".pcornet_lab -with lab_map as ( - select distinct lab.c_hlevel, lab.c_path, lab.pcori_specimen_source, trim(CHR(13) from lab.pcori_basecode) as pcori_basecode - from "&&i2b2_meta_schema".pcornet_lab lab - inner JOIN "&&i2b2_meta_schema".pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname - inner join pmn_labnormal norm on ont_parent.c_basecode=norm.LAB_NAME - where lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' -), -local_loinc_terms as ( - select lm.C_HLEVEL, lt.C_FULLNAME, lm.C_PATH, lm.PCORI_BASECODE, lm.pcori_specimen_source - from "&&i2b2_meta_schema"."&&terms_table" lt, lab_map lm - where lm.pcori_basecode=replace(lt.c_basecode, 'LOINC:', '') - and lt.c_fullname like '\i2b2\Laboratory Tests\%' and lt.c_basecode like 'LOINC:%' +with parent_loinc_codes as ( -- LOINC codes with children LOINC codes + select p_loinc.* + from "&&i2b2_meta_schema"."&&terms_table" p_loinc + join "&&i2b2_meta_schema"."&&terms_table" c_loinc + on c_loinc.c_fullname like (p_loinc.c_fullname || '%') + and p_loinc.c_basecode like 'LOINC:%' + and c_loinc.c_basecode like 'LOINC:%' + and p_loinc.c_basecode!=c_loinc.c_basecode ) -select - llt.C_HLEVEL, - concat(llt.c_path, substr(regexp_substr(lt.c_fullname, '\\[^\\]+\\$'), 2, length(regexp_substr(lt.c_fullname, '\\[^\\]+\\$')))) as c_fullname, - lt.C_NAME, - lt.C_SYNONYM_CD, - lt.C_VISUALATTRIBUTES, - lt.C_TOTALNUM, - lt.C_BASECODE, - lt.C_METADATAXML, - lt.C_FACTTABLECOLUMN, - lt.C_TABLENAME, - lt.C_COLUMNNAME, - lt.C_COLUMNDATATYPE, - lt.C_OPERATOR, - lt.C_DIMCODE, - lt.C_COMMENT, - lt.C_TOOLTIP, - lt.M_APPLIED_PATH, - lt.UPDATE_DATE, - lt.DOWNLOAD_DATE, - lt.IMPORT_DATE, - 'MAPPING', - lt.VALUETYPE_CD, - lt.M_EXCLUSION_CD, - llt.C_PATH, - lt.C_SYMBOL, - llt.pcori_specimen_source, - llt.PCORI_BASECODE -from "&&i2b2_meta_schema"."&&terms_table" lt, local_loinc_terms llt -where lt.c_fullname like llt.c_fullname||'%\' +, children_loinc_codes as ( -- LOINC codes without children LOINC codes + select clc.* + from "&&i2b2_meta_schema"."&&terms_table" clc + where clc.c_basecode like 'LOINC:%' + and clc.c_basecode not in ( + select c_basecode from parent_loinc_codes + ) +) +select + ccc.C_HLEVEL, + replace(ccc.C_FULLNAME, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_fullname, + ccc.C_NAME, + ccc.C_SYNONYM_CD, + ccc.C_VISUALATTRIBUTES, + ccc.C_TOTALNUM, + ccc.C_BASECODE, + ccc.C_METADATAXML, + ccc.C_FACTTABLECOLUMN, + ccc.C_TABLENAME, + ccc.C_COLUMNNAME, + ccc.C_COLUMNDATATYPE, + ccc.C_OPERATOR, + ccc.C_DIMCODE, + ccc.C_COMMENT, + ccc.C_TOOLTIP, + ccc.M_APPLIED_PATH, + ccc.UPDATE_DATE, + ccc.DOWNLOAD_DATE, + ccc.IMPORT_DATE, + ccc.SOURCESYSTEM_CD, + ccc.VALUETYPE_CD, + ccc.M_EXCLUSION_CD, + replace(clc.C_FULLNAME, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_path, + ccc.C_SYMBOL, + clc.pcori_specimen_source, + replace(clc.c_basecode, 'LOINC:', '') pcori_basecode +from children_loinc_codes clc +join "&&i2b2_meta_schema"."&&terms_table" ccc + on ccc.c_fullname like (clc.c_fullname || '%') + and ccc.c_basecode like 'KUH|COMPONENT_ID:%' -- TODO: Generalize for other sites. ; commit; diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 2817345..f0a4b7b 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -25,6 +25,7 @@ set -e python load_csv.py harvest_local harvest_local.csv harvest_local.ctl pcornet_cdm_user pcornet_cdm python load_csv.py PMN_LabNormal pmn_labnormal.csv pmn_labnormal.ctl pcornet_cdm_user pcornet_cdm +python load_csv.py lab_loinc_mapping lab_loinc_mapping.csv lab_loinc_mapping.ctl pcornet_cdm_user pcornet_cdm . ./load_pcornet_mapping.sh # Run some tests From 5e58d5287bfe3ec242966f88dc2cc94880a4eff4 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 11 Apr 2017 10:49:22 -0500 Subject: [PATCH 233/507] A few small fixes --- Oracle/PCORNetLoader_ora.sql | 2 +- Oracle/pcornet_mapping.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 99101f9..a4f304d 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1397,7 +1397,7 @@ INSERT INTO lab_result_cm SELECT DISTINCT M.patient_num patid, M.encounter_num encounterid, -CASE WHEN ont_parent.C_BASECODE LIKE 'LAB_NAME%' then SUBSTR (ont_parent.c_basecode,10, 10) ELSE 'NI' END LAB_NAME, +CASE WHEN ont_parent.C_BASECODE LIKE 'LAB_NAME%' then SUBSTR (ont_parent.c_basecode,10, 10) ELSE null END LAB_NAME, CASE WHEN lab.pcori_specimen_source like '%or SR_PLS' THEN 'SR_PLS' WHEN lab.pcori_specimen_source is null then 'NI' ELSE lab.pcori_specimen_source END specimen_source, -- (Better way would be to fix the column in the ontology but this will work) NVL(lab.pcori_basecode, 'NI') LAB_LOINC, NVL(p.PRIORITY,'NI') PRIORITY, diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 5a0ecfa..a854429 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -601,7 +601,7 @@ select case when llm.lab_name is not null then 'LAB_NAME:'|| llm.lab_name else lc.c_basecode - end c_basecose, + end c_basecode, lc.C_METADATAXML, lc.C_FACTTABLECOLUMN, lc.C_TABLENAME, From 8330aa9d79a9e91337eb994ad0aab6066e633c54 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 11 Apr 2017 10:52:03 -0500 Subject: [PATCH 234/507] Added distinct to select to make sure that dispensing rows are not duplicates --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 1af39be..ac0d2c2 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1781,7 +1781,7 @@ with disp_status as ( on ibf.modifier_cd=pnm.c_basecode where pnm.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\%' ) -select +select distinct st.patient_num patid, null prescribingid, st.start_date dispense_date, From cd76d7094e7e96bb1710fb073fdd90b3a867f152 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 11 Apr 2017 10:53:47 -0500 Subject: [PATCH 235/507] Removed commented out code --- Oracle/PCORNetLoader_ora.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index a4f304d..e9969f2 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1453,7 +1453,6 @@ and M.concept_cd=l.concept_Cd and M.start_date=l.start_Date WHERE m.ValType_Cd in ('N','T') ---and ont_parent.C_BASECODE LIKE 'LAB_NAME%' -- Exclude non-pcori labs and m.MODIFIER_CD='@'; execute immediate 'create index lab_result_cm_patid on lab_result_cm (PATID)'; From f7284529f7627b30d57419e63e187aaa7f6f1715 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 11 Apr 2017 13:25:54 -0500 Subject: [PATCH 236/507] Fixed error with populating specimen source --- Oracle/pcornet_mapping.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index a854429..5edfe5d 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -674,12 +674,14 @@ select ccc.M_EXCLUSION_CD, replace(clc.C_FULLNAME, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_path, ccc.C_SYMBOL, - clc.pcori_specimen_source, + llm.pcori_specimen_source, replace(clc.c_basecode, 'LOINC:', '') pcori_basecode from children_loinc_codes clc join "&&i2b2_meta_schema"."&&terms_table" ccc on ccc.c_fullname like (clc.c_fullname || '%') and ccc.c_basecode like 'KUH|COMPONENT_ID:%' -- TODO: Generalize for other sites. +left outer join mprittie.lab_loinc_mapping llm + on clc.c_basecode = ('LOINC:' || llm.loinc_code) ; commit; From d386ef222eefc5c66f55d92f19dbd9e53bbef8a5 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 11 Apr 2017 15:11:41 -0500 Subject: [PATCH 237/507] Removed reference to testing schema --- Oracle/pcornet_mapping.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 5edfe5d..7f64922 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -680,7 +680,7 @@ from children_loinc_codes clc join "&&i2b2_meta_schema"."&&terms_table" ccc on ccc.c_fullname like (clc.c_fullname || '%') and ccc.c_basecode like 'KUH|COMPONENT_ID:%' -- TODO: Generalize for other sites. -left outer join mprittie.lab_loinc_mapping llm +left outer join lab_loinc_mapping llm on clc.c_basecode = ('LOINC:' || llm.loinc_code) ; From 91b75e50cb80cc23b435c4cda4da7480715c08d8 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 11 Apr 2017 16:24:56 -0500 Subject: [PATCH 238/507] Maximum field length fix --- Oracle/PCORNetLoader_ora.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index e9969f2..bb29203 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1414,7 +1414,12 @@ to_char(m.end_date,'HH:MI') RESULT_TIME, CASE WHEN m.ValType_Cd='N' THEN m.NVAL_NUM ELSE null END RESULT_NUM, CASE WHEN m.ValType_Cd='N' THEN (CASE NVL(nullif(m.TVal_Char,''),'NI') WHEN 'E' THEN 'EQ' WHEN 'NE' THEN 'OT' WHEN 'L' THEN 'LT' WHEN 'LE' THEN 'LE' WHEN 'G' THEN 'GT' WHEN 'GE' THEN 'GE' ELSE 'NI' END) ELSE 'TX' END RESULT_MODIFIER, --NVL(m.Units_CD,'NI') RESULT_UNIT, -- TODO: Should be standardized units -CASE WHEN INSTR(m.Units_CD, '%') > 0 THEN 'PERCENT' WHEN m.Units_CD IS NULL THEN NVL(m.Units_CD,'NI') ELSE TRIM(REPLACE(UPPER(m.Units_CD), '(CALC)', '')) end RESULT_UNIT, -- Local fix for KUMC +CASE + WHEN INSTR(m.Units_CD, '%') > 0 THEN 'PERCENT' + WHEN m.Units_CD IS NULL THEN NVL(m.Units_CD,'NI') + when length(m.Units_CD) > 11 then substr(m.Units_CD, 1, 11) + ELSE TRIM(REPLACE(UPPER(m.Units_CD), '(CALC)', '')) +end RESULT_UNIT, -- Local fix for KUMC nullif(norm.NORM_RANGE_LOW,'') NORM_RANGE_LOW ,norm.NORM_MODIFIER_LOW, nullif(norm.NORM_RANGE_HIGH,'') NORM_RANGE_HIGH From ca4f33228a675eb0382702875c6320c852fdfb45 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 12 Apr 2017 09:01:54 -0500 Subject: [PATCH 239/507] Hotfix for dispensing procedure error --- Oracle/PCORNetLoader_ora.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index ac0d2c2..556d4ec 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1765,7 +1765,7 @@ with disp_status as ( where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' /* TODO: Generalize for other sites. The '< 12' makes sure only 11 digit codes are included. */ - and length(replace(concept_cd, 'NDC:', '')) < 12 + and length(replace(ibf.concept_cd, 'NDC:', '')) < 12 ) , disp_quantity as ( select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num @@ -1785,7 +1785,7 @@ select distinct st.patient_num patid, null prescribingid, st.start_date dispense_date, - replace(concept_cd, 'NDC:', '') ndc, -- TODO: Generalize this for other sites. + replace(st.concept_cd, 'NDC:', '') ndc, -- TODO: Generalize this for other sites. ds.nval_num dispense_sup, qt.nval_num dispense_amt from disp_status st From aea362eaa40fc686c322de1442937e212210610e Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 13 Apr 2017 11:14:06 -0500 Subject: [PATCH 240/507] Add KUH|COMPONENT_ID to raw field in labs --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 2b40dea..b986e88 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1432,7 +1432,7 @@ NULL RAW_PANEL, CASE WHEN m.ValType_Cd='T' THEN substr(m.TVal_Char, 1, 50) ELSE to_char(m.NVal_Num) END RAW_RESULT, -- Local fix for KUMC NULL RAW_UNIT, NULL RAW_ORDER_DEPT, -NULL RAW_FACILITY_CODE +NULL m.concept_cd RAW_FACILITY_CODE FROM i2b2fact M inner join encounter enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num -- Constraint to selected encounters From f66a9b68f670eb2b5d0144e5c258b040c445677c Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 13 Apr 2017 11:14:48 -0500 Subject: [PATCH 241/507] Make sure DISPENSING rows are distinct --- Oracle/pcornet_mapping.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 7f64922..52b6daf 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -591,7 +591,7 @@ truncate table "&&i2b2_meta_schema".pcornet_lab; /* Insert LOINC codes from local hierarchy, setting c_basecode to appropriate LAB_NAME for common PCORNet labs. */ insert into "&&i2b2_meta_schema".pcornet_lab -select +select distinct lc.C_HLEVEL, replace(lc.C_FULLNAME, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_fullname, lc.C_NAME, @@ -648,7 +648,7 @@ with parent_loinc_codes as ( -- LOINC codes with children LOINC codes select c_basecode from parent_loinc_codes ) ) -select +select distinct ccc.C_HLEVEL, replace(ccc.C_FULLNAME, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_fullname, ccc.C_NAME, From 66afd710b416bcf3fc8bcd39f09eae93176e1f6a Mon Sep 17 00:00:00 2001 From: Matt Hoag Date: Thu, 13 Apr 2017 11:17:55 -0500 Subject: [PATCH 242/507] Identify CDM IP based on UHC LOS instead of IDX IP flag. --- Oracle/pcornet_mapping.csv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index fee21ee..194191a 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -34,6 +34,7 @@ pcori_path,local_path \PCORI\ENCOUNTER\ENC_TYPE\OA\,\i2b2\Visit Details\ENC_TYPE\OA\ \PCORI\ENCOUNTER\ENC_TYPE\OT\,\i2b2\Visit Details\ENC_TYPE\OT\ \PCORI\ENCOUNTER\ENC_TYPE\UN\,\i2b2\Visit Details\ENC_TYPE\UN\ +\PCORI\ENCOUNTER\ENC_TYPE\IP\,\i2b2\UHC\Visit Details\Length of Stay\Hospital\ \PCORI\DEMOGRAPHIC\SEX\F\,\i2b2\Demographics\Gender\Female\ \PCORI\DEMOGRAPHIC\SEX\M\,\i2b2\Demographics\Gender\Male\ \PCORI\DEMOGRAPHIC\SEX\NI\,\i2b2\Demographics\Gender\Unknown\Unknown-@\ @@ -191,7 +192,7 @@ pcori_path,local_path "\PCORI\ENCOUNTER\ENC_TYPE\OA\","\i2b2\Visit Details\Place of Service (IDX)\03\" "\PCORI\ENCOUNTER\ENC_TYPE\AV\","\i2b2\Visit Details\Place of Service (IDX)\26\" "\PCORI\ENCOUNTER\ENC_TYPE\AV\","\i2b2\Visit Details\Place of Service (IDX)\11\" -"\PCORI\ENCOUNTER\ENC_TYPE\IP\","\i2b2\Visit Details\Place of Service (IDX)\21\" +"\PCORI\ENCOUNTER\ENC_TYPE\OA\","\i2b2\Visit Details\Place of Service (IDX)\21\" "\PCORI\ENCOUNTER\ENC_TYPE\ED\","\i2b2\Visit Details\Place of Service (IDX)\23\" "\PCORI\ENCOUNTER\ENC_TYPE\IS\","\i2b2\Visit Details\Place of Service (IDX)\13\" "\PCORI\ENCOUNTER\ENC_TYPE\IS\","\i2b2\Visit Details\Place of Service (IDX)\65\" From 6298bd96fbbf57774dadbd42ca4030209b3b2949 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 13 Apr 2017 15:36:40 -0500 Subject: [PATCH 243/507] Hotfix for select distinct / CLOB errors --- Oracle/pcornet_mapping.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 52b6daf..743248c 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -602,14 +602,14 @@ select distinct when llm.lab_name is not null then 'LAB_NAME:'|| llm.lab_name else lc.c_basecode end c_basecode, - lc.C_METADATAXML, + to_char(lc.C_METADATAXML), lc.C_FACTTABLECOLUMN, lc.C_TABLENAME, lc.C_COLUMNNAME, lc.C_COLUMNDATATYPE, lc.C_OPERATOR, lc.C_DIMCODE, - lc.C_COMMENT, + to_char(lc.C_COMMENT), lc.C_TOOLTIP, lc.M_APPLIED_PATH, lc.UPDATE_DATE, @@ -656,14 +656,14 @@ select distinct ccc.C_VISUALATTRIBUTES, ccc.C_TOTALNUM, ccc.C_BASECODE, - ccc.C_METADATAXML, + to_char(ccc.C_METADATAXML), ccc.C_FACTTABLECOLUMN, ccc.C_TABLENAME, ccc.C_COLUMNNAME, ccc.C_COLUMNDATATYPE, ccc.C_OPERATOR, ccc.C_DIMCODE, - ccc.C_COMMENT, + to_char(ccc.C_COMMENT), ccc.C_TOOLTIP, ccc.M_APPLIED_PATH, ccc.UPDATE_DATE, From ed0fb77468bd48e4945d61ec5945b64e98916e04 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 13 Apr 2017 16:03:12 -0500 Subject: [PATCH 244/507] Hotfixed typo --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index b986e88..10c7a99 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1432,7 +1432,7 @@ NULL RAW_PANEL, CASE WHEN m.ValType_Cd='T' THEN substr(m.TVal_Char, 1, 50) ELSE to_char(m.NVal_Num) END RAW_RESULT, -- Local fix for KUMC NULL RAW_UNIT, NULL RAW_ORDER_DEPT, -NULL m.concept_cd RAW_FACILITY_CODE +m.concept_cd RAW_FACILITY_CODE FROM i2b2fact M inner join encounter enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num -- Constraint to selected encounters From 9092891342e43ab7ea8a55b5d0c986625f2093fc Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 14 Apr 2017 09:12:54 -0500 Subject: [PATCH 245/507] don't exclude EI, UHC when looking for PDX=P Though the CDM spec says "Relevant only on IP and IS encounters," table Table IVE. Principal Diagnoses for Institutional Encounters of the EDC report includes a row for EI as well. We raised https://github.com/CDMFORUM/CDM-ERRATA/issues/19 about this. Meanwhile, we have been counting only Epic billing diagnoses as discharge diagnoses when looking for PDX. The `\Diagnosis\Billing\Primary\` modifier used in UHC should be counted as well. This modifier is also used in IDX, which will raise our count of "For ED, AV, and OA encounter types, mark as X=Unable to Classify." but that seems worthwhile. --- Oracle/PCORNetLoader_ora.sql | 2 +- Oracle/pcornet_mapping.csv | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 10c7a99..f58ff44 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1051,7 +1051,7 @@ select distinct factline.patient_num, factline.encounter_num encounterid, enc_ty else '09' end dxtype, CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END dx_source, - CASE WHEN enc_type in ('IP', 'IS') -- PDX is "relevant only on IP and IS encounters" + CASE WHEN enc_type in ('EI', 'IP', 'IS') -- PDX is "relevant only on IP and IS encounters" THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, ':')+1,2),'NI') ELSE 'X' END PDX from i2b2fact factline diff --git a/Oracle/pcornet_mapping.csv b/Oracle/pcornet_mapping.csv index 8fb8e6f..8d7e373 100644 --- a/Oracle/pcornet_mapping.csv +++ b/Oracle/pcornet_mapping.csv @@ -7,6 +7,7 @@ pcori_path,local_path \PCORI_MOD\RX_BASIS\PR\01\,\Medication\Outpatient\ \PCORI_MOD\RX_BASIS\PR\02\,\Medication\PRN Inpatient Order\ \PCORI_MOD\PDX\P\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\discharge_primary\ +\PCORI_MOD\PDX\P\,\Diagnosis\Billing\Primary\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\DI\,\Diagnosis\\modifier_dx\billing_diagnosis\discharge\ \PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\AD\,\Diagnosis\\modifier_dx\billing_diagnosis\admit\ "\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\UN\","\Diagnosis\Billing\UHC_DIAGNOSIS\" From d8054ccdc97e1862540226f74a7736259214f7f0 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 14 Apr 2017 14:54:00 -0500 Subject: [PATCH 246/507] Don't exclude FN (i.e. VA classes) --- Oracle/pcornet_mapping.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 9635f53..23eb344 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -417,7 +417,6 @@ with pcornet_spec as ( select clarity_medication_id, rxcui, '4) Manual Curation' --, con.tty, va_name, sdf_name from heron_etl_1.med_map_manual_curation mmmc join rxnorm.rxnconso@id con on con.rxaui = mmmc.sdf_rxaui - where con.tty not in ('FN') ) , med_map_pref as ( -- HERON clarity_med_id_to_rxcui joined with spec order select distinct clarity_med_id medication_id From c413bf5a188ae7a17362d399fe73fef957359456 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 14 Apr 2017 16:32:44 -0500 Subject: [PATCH 247/507] oops: fix med id type (#76) and un-hard-code ETL schema --- Oracle/pcornet_mapping.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index e6523bc..f373b9a 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -414,8 +414,8 @@ with pcornet_spec as ( union all - select clarity_medication_id, rxcui, '4) Manual Curation' --, con.tty, va_name, sdf_name - from heron_etl_1.med_map_manual_curation mmmc + select to_number(clarity_medication_id), rxcui, '4) Manual Curation' --, con.tty, va_name, sdf_name + from "&&i2b2_etl_schema".med_map_manual_curation mmmc join rxnorm.rxnconso@id con on con.rxaui = mmmc.sdf_rxaui ) , med_map_pref as ( -- HERON clarity_med_id_to_rxcui joined with spec order From 806692042468abec4c94208bc83504f0c2a6fd03 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 17 Apr 2017 08:24:56 -0500 Subject: [PATCH 248/507] don't add blank lines; sqlplus doesn't grok --- Oracle/pcornet_mapping.sql | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index f373b9a..eeb2a3f 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -401,19 +401,13 @@ with pcornet_spec as ( select cmed.medication_id clarity_med_id, rxn.rxnorm_code rxcui, '1) Clarity' dose_pref from clarity.rxnorm_codes@id rxn join clarity.clarity_medication@id cmed on cmed.medication_id = rxn.medication_id - union all - select clarity_med_id, rxcui, '2) GCN' from "&&i2b2_etl_schema".clarity_med_id_to_rxcui_gcn@id - union all - select clarity_med_id, rxcui, '3) NDC' from "&&i2b2_etl_schema".clarity_med_id_to_rxcui_ndc@id - union all - select to_number(clarity_medication_id), rxcui, '4) Manual Curation' --, con.tty, va_name, sdf_name from "&&i2b2_etl_schema".med_map_manual_curation mmmc join rxnorm.rxnconso@id con on con.rxaui = mmmc.sdf_rxaui From 26d47db0323a36382bc1cf33a57e25a3a79b8cdf Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 17 Apr 2017 17:33:30 -0500 Subject: [PATCH 249/507] pcornet_mapping: optimize LOINC mapping - compute children_loinc_codes directly without first computing parent_loinc_codes - use c_hlevel to cut down parent / child search space - factor out - lab_terms: where c_fullname like '\i2b2\Laboratory Tests\%' - loinc_terms: where c_basecode like 'LOINC:%' - use just 3 columns for the tricky parent/child part - Oracle seems to build a temp table for this Fetched 50 rows in 122.732 seconds --- Oracle/pcornet_mapping.sql | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index eeb2a3f..6b32c77 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -635,21 +635,23 @@ where lc.c_basecode like 'LOINC:%' /* Insert child KUH|COMPONENT_ID nodes, setting pcori_basecode, and c_path to that of it's parent. */ insert into "&&i2b2_meta_schema".pcornet_lab -with parent_loinc_codes as ( -- LOINC codes with children LOINC codes - select p_loinc.* - from "&&i2b2_meta_schema"."&&terms_table" p_loinc - join "&&i2b2_meta_schema"."&&terms_table" c_loinc - on c_loinc.c_fullname like (p_loinc.c_fullname || '%') - and p_loinc.c_basecode like 'LOINC:%' - and c_loinc.c_basecode like 'LOINC:%' - and p_loinc.c_basecode!=c_loinc.c_basecode +with lab_terms as ( + select c_fullname, c_hlevel, c_basecode + from "&&i2b2_meta_schema"."&&terms_table" + where c_fullname like '\i2b2\Laboratory Tests\%' +) +, loinc_terms as ( + select * + from lab_terms + where c_basecode like 'LOINC:%' ) , children_loinc_codes as ( -- LOINC codes without children LOINC codes select clc.* - from "&&i2b2_meta_schema"."&&terms_table" clc - where clc.c_basecode like 'LOINC:%' - and clc.c_basecode not in ( - select c_basecode from parent_loinc_codes + from loinc_terms clc + where not exists ( + select 1 from loinc_terms p + where p.c_hlevel < clc.c_hlevel + and clc.c_fullname like (p.c_fullname || '%') ) ) select distinct @@ -660,7 +662,7 @@ select distinct ccc.C_VISUALATTRIBUTES, ccc.C_TOTALNUM, ccc.C_BASECODE, - to_char(ccc.C_METADATAXML), +-- to_char(ccc.C_METADATAXML), ccc.C_FACTTABLECOLUMN, ccc.C_TABLENAME, ccc.C_COLUMNNAME, @@ -681,9 +683,12 @@ select distinct llm.pcori_specimen_source, replace(clc.c_basecode, 'LOINC:', '') pcori_basecode from children_loinc_codes clc -join "&&i2b2_meta_schema"."&&terms_table" ccc - on ccc.c_fullname like (clc.c_fullname || '%') - and ccc.c_basecode like 'KUH|COMPONENT_ID:%' -- TODO: Generalize for other sites. +join (select * from "&&i2b2_meta_schema"."&&terms_table" + where c_fullname like '\i2b2\Laboratory Tests\%' + and c_basecode like 'KUH|COMPONENT_ID:%' -- TODO: Generalize for other sites. + ) ccc + on ccc.c_hlevel > clc.c_hlevel + and ccc.c_fullname like (clc.c_fullname || '%') left outer join lab_loinc_mapping llm on clc.c_basecode = ('LOINC:' || llm.loinc_code) ; From 8485cac388b40873a56764911eaec2d6fc5a2081 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 17 Apr 2017 18:03:22 -0500 Subject: [PATCH 250/507] oops: can't just leave c_metadataxml out --- Oracle/pcornet_mapping.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 6b32c77..4369b3c 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -662,7 +662,7 @@ select distinct ccc.C_VISUALATTRIBUTES, ccc.C_TOTALNUM, ccc.C_BASECODE, --- to_char(ccc.C_METADATAXML), + null C_METADATAXML, -- to_char(ccc.C_METADATAXML), ccc.C_FACTTABLECOLUMN, ccc.C_TABLENAME, ccc.C_COLUMNNAME, From bc008cede345a41c703c235a89fb4bccdbfc6344 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 19 Apr 2017 15:27:34 -0500 Subject: [PATCH 251/507] Remove rows from PRESCRIBING where all RX_* fields are blank. --- Oracle/cdm_postproc.sql | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Oracle/cdm_postproc.sql b/Oracle/cdm_postproc.sql index 57c4a9d..54a26d2 100644 --- a/Oracle/cdm_postproc.sql +++ b/Oracle/cdm_postproc.sql @@ -23,4 +23,15 @@ when matched then update set p.rx_providerid = e.providerid; /* Currently in HERON, we have hight in cm and weight in oz (from visit vitals). The CDM wants height in inches and weight in pounds. */ update vital v set v.ht = v.ht / 2.54; -update vital v set v.wt = v.wt / 16; \ No newline at end of file +update vital v set v.wt = v.wt / 16; + +/* Remove rows from the PRESCRIBING table where RX_* fields are null + TODO: Remove this when fixed in HERON + */ +delete +from pcornet_cdm.prescribing +where rx_basis is null + and rx_quantity is null + and rx_frequency is null + and rx_refills is null +; \ No newline at end of file From aa9455ccd9aff0ed8eeba2a3bfdbb461bcd9d5dc Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Thu, 20 Apr 2017 14:31:23 -0500 Subject: [PATCH 252/507] Revert "pcornet_mapping: optimize LOINC mapping" This reverts commit 26d47db0323a36382bc1cf33a57e25a3a79b8cdf. --- Oracle/pcornet_mapping.sql | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/Oracle/pcornet_mapping.sql b/Oracle/pcornet_mapping.sql index 4369b3c..eeb2a3f 100644 --- a/Oracle/pcornet_mapping.sql +++ b/Oracle/pcornet_mapping.sql @@ -635,23 +635,21 @@ where lc.c_basecode like 'LOINC:%' /* Insert child KUH|COMPONENT_ID nodes, setting pcori_basecode, and c_path to that of it's parent. */ insert into "&&i2b2_meta_schema".pcornet_lab -with lab_terms as ( - select c_fullname, c_hlevel, c_basecode - from "&&i2b2_meta_schema"."&&terms_table" - where c_fullname like '\i2b2\Laboratory Tests\%' -) -, loinc_terms as ( - select * - from lab_terms - where c_basecode like 'LOINC:%' +with parent_loinc_codes as ( -- LOINC codes with children LOINC codes + select p_loinc.* + from "&&i2b2_meta_schema"."&&terms_table" p_loinc + join "&&i2b2_meta_schema"."&&terms_table" c_loinc + on c_loinc.c_fullname like (p_loinc.c_fullname || '%') + and p_loinc.c_basecode like 'LOINC:%' + and c_loinc.c_basecode like 'LOINC:%' + and p_loinc.c_basecode!=c_loinc.c_basecode ) , children_loinc_codes as ( -- LOINC codes without children LOINC codes select clc.* - from loinc_terms clc - where not exists ( - select 1 from loinc_terms p - where p.c_hlevel < clc.c_hlevel - and clc.c_fullname like (p.c_fullname || '%') + from "&&i2b2_meta_schema"."&&terms_table" clc + where clc.c_basecode like 'LOINC:%' + and clc.c_basecode not in ( + select c_basecode from parent_loinc_codes ) ) select distinct @@ -662,7 +660,7 @@ select distinct ccc.C_VISUALATTRIBUTES, ccc.C_TOTALNUM, ccc.C_BASECODE, - null C_METADATAXML, -- to_char(ccc.C_METADATAXML), + to_char(ccc.C_METADATAXML), ccc.C_FACTTABLECOLUMN, ccc.C_TABLENAME, ccc.C_COLUMNNAME, @@ -683,12 +681,9 @@ select distinct llm.pcori_specimen_source, replace(clc.c_basecode, 'LOINC:', '') pcori_basecode from children_loinc_codes clc -join (select * from "&&i2b2_meta_schema"."&&terms_table" - where c_fullname like '\i2b2\Laboratory Tests\%' - and c_basecode like 'KUH|COMPONENT_ID:%' -- TODO: Generalize for other sites. - ) ccc - on ccc.c_hlevel > clc.c_hlevel - and ccc.c_fullname like (clc.c_fullname || '%') +join "&&i2b2_meta_schema"."&&terms_table" ccc + on ccc.c_fullname like (clc.c_fullname || '%') + and ccc.c_basecode like 'KUH|COMPONENT_ID:%' -- TODO: Generalize for other sites. left outer join lab_loinc_mapping llm on clc.c_basecode = ('LOINC:' || llm.loinc_code) ; From cd330487be6f02e174dff68fc437bf22b8b65ffd Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 11 May 2017 14:08:49 -0500 Subject: [PATCH 253/507] Removed mapping pieces and drop tables statments from shell/sqlplus script --- Oracle/run-i2p-transform.sh | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index f0a4b7b..ac7967e 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -26,7 +26,6 @@ set -e python load_csv.py harvest_local harvest_local.csv harvest_local.ctl pcornet_cdm_user pcornet_cdm python load_csv.py PMN_LabNormal pmn_labnormal.csv pmn_labnormal.ctl pcornet_cdm_user pcornet_cdm python load_csv.py lab_loinc_mapping lab_loinc_mapping.csv lab_loinc_mapping.ctl pcornet_cdm_user pcornet_cdm -. ./load_pcornet_mapping.sh # Run some tests sqlplus /nolog < Date: Tue, 16 May 2017 15:43:47 -0500 Subject: [PATCH 254/507] Made PATID and ENCOUNTERID fields numeric for more performant joins. --- Oracle/PCORNetLoader_ora.sql | 44 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index f58ff44..78ad564 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -249,7 +249,7 @@ END; / CREATE TABLE enrollment ( - PATID varchar(50) NOT NULL, + PATID number(38,0) NOT NULL, ENR_START_DATE date NOT NULL, ENR_END_DATE date NULL, CHART varchar(1) NULL, @@ -268,8 +268,8 @@ END; CREATE TABLE vital ( VITALID varchar(19) primary key, - PATID varchar(50) NULL, - ENCOUNTERID varchar(50) NULL, + PATID number(38,0) NULL, + ENCOUNTERID number(38,0) NULL, MEASURE_DATE date NULL, MEASURE_TIME varchar(5) NULL, VITAL_SOURCE varchar(2) NULL, @@ -318,8 +318,8 @@ END; CREATE TABLE procedures( PROCEDURESID varchar(19) primary key, - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NOT NULL, + PATID number(38,0) NOT NULL, + ENCOUNTERID number(38,0) NOT NULL, ENC_TYPE varchar(2) NULL, ADMIT_DATE date NULL, PROVIDERID varchar(50) NULL, @@ -355,8 +355,8 @@ END; CREATE TABLE diagnosis( DIAGNOSISID varchar(19) primary key, - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NOT NULL, + PATID number(38,0) NOT NULL, + ENCOUNTERID number(38,0) NOT NULL, ENC_TYPE varchar(2) NULL, ADMIT_DATE date NULL, PROVIDERID varchar(50) NULL, @@ -397,8 +397,8 @@ END; CREATE TABLE lab_result_cm( LAB_RESULT_CM_ID varchar(19) primary key, - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NULL, + PATID number(38,0) NOT NULL, + ENCOUNTERID number(38,0) NULL, LAB_NAME varchar(10) NULL, SPECIMEN_SOURCE varchar(10) NULL, LAB_LOINC varchar(10) NULL, @@ -453,7 +453,7 @@ PMN_DROPSQL('DROP TABLE death'); END; / CREATE TABLE death( - PATID varchar(50) NOT NULL, + PATID number(38,0) NOT NULL, DEATH_DATE date NOT NULL, DEATH_DATE_IMPUTE varchar(2) NULL, DEATH_SOURCE varchar(2) NOT NULL, @@ -466,7 +466,7 @@ PMN_DROPSQL('DROP TABLE death_cause'); END; / CREATE TABLE death_cause( - PATID varchar(50) NOT NULL, + PATID number(38,0) NOT NULL, DEATH_CAUSE varchar(8) NOT NULL, DEATH_CAUSE_CODE varchar(2) NOT NULL, DEATH_CAUSE_TYPE varchar(2) NOT NULL, @@ -481,7 +481,7 @@ END; / CREATE TABLE dispensing( DISPENSINGID varchar(19) primary key, - PATID varchar(50) NOT NULL, + PATID number(38,0) NOT NULL, PRESCRIBINGID varchar(19) NULL, DISPENSE_DATE date NOT NULL, NDC varchar (11) NOT NULL, @@ -522,8 +522,8 @@ END; / CREATE TABLE prescribing( PRESCRIBINGID varchar(19) primary key, - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NULL, + PATID number(38,0) NOT NULL, + ENCOUNTERID number(38,0) NULL, RX_PROVIDERID varchar(50) NULL, -- NOTE: The spec has a _ before the ID, but this is inconsistent. RX_ORDER_DATE date NULL, RX_ORDER_TIME varchar (5) NULL, @@ -562,7 +562,7 @@ PMN_DROPSQL('DROP TABLE pcornet_trial'); END; / CREATE TABLE pcornet_trial( - PATID varchar(50) NOT NULL, + PATID number(38,0) NOT NULL, TRIALID varchar(20) NOT NULL, PARTICIPANTID varchar(50) NOT NULL, TRIAL_SITEID varchar(50) NULL, @@ -579,8 +579,8 @@ END; / CREATE TABLE condition( CONDITIONID varchar(19) primary key, - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NULL, + PATID number(38,0) NOT NULL, + ENCOUNTERID number(38,0) NULL, REPORT_DATE date NULL, RESOLVE_DATE date NULL, ONSET_DATE date NULL, @@ -618,8 +618,8 @@ END; / CREATE TABLE pro_cm( PRO_CM_ID varchar(19) primary key, - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NULL, + PATID number(38,0) NOT NULL, + ENCOUNTERID number(38,0) NULL, PRO_ITEM varchar (7) NOT NULL, PRO_LOINC varchar (10) NULL, PRO_DATE date NOT NULL, @@ -703,8 +703,8 @@ PMN_DROPSQL('DROP TABLE encounter'); END; / CREATE TABLE encounter( - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NOT NULL, + PATID number(38,0) NOT NULL, + ENCOUNTERID number(38,0) NOT NULL, ADMIT_DATE date NULL, ADMIT_TIME varchar(5) NULL, DISCHARGE_DATE date NULL, @@ -732,7 +732,7 @@ PMN_DROPSQL('DROP TABLE demographic'); END; / CREATE TABLE demographic( - PATID varchar(50) NOT NULL, + PATID number(38,0) NOT NULL, BIRTH_DATE date NULL, BIRTH_TIME varchar(5) NULL, SEX varchar(2) NULL, From 49ddf1a3d2ecbab86190a7daaaacf778baa0dbf1 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 18 May 2017 09:50:37 -0500 Subject: [PATCH 255/507] Undoing PATID and ENCOUNTERID field types as this belongs on another branch --- Oracle/PCORNetLoader_ora.sql | 44 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 78ad564..f58ff44 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -249,7 +249,7 @@ END; / CREATE TABLE enrollment ( - PATID number(38,0) NOT NULL, + PATID varchar(50) NOT NULL, ENR_START_DATE date NOT NULL, ENR_END_DATE date NULL, CHART varchar(1) NULL, @@ -268,8 +268,8 @@ END; CREATE TABLE vital ( VITALID varchar(19) primary key, - PATID number(38,0) NULL, - ENCOUNTERID number(38,0) NULL, + PATID varchar(50) NULL, + ENCOUNTERID varchar(50) NULL, MEASURE_DATE date NULL, MEASURE_TIME varchar(5) NULL, VITAL_SOURCE varchar(2) NULL, @@ -318,8 +318,8 @@ END; CREATE TABLE procedures( PROCEDURESID varchar(19) primary key, - PATID number(38,0) NOT NULL, - ENCOUNTERID number(38,0) NOT NULL, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NOT NULL, ENC_TYPE varchar(2) NULL, ADMIT_DATE date NULL, PROVIDERID varchar(50) NULL, @@ -355,8 +355,8 @@ END; CREATE TABLE diagnosis( DIAGNOSISID varchar(19) primary key, - PATID number(38,0) NOT NULL, - ENCOUNTERID number(38,0) NOT NULL, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NOT NULL, ENC_TYPE varchar(2) NULL, ADMIT_DATE date NULL, PROVIDERID varchar(50) NULL, @@ -397,8 +397,8 @@ END; CREATE TABLE lab_result_cm( LAB_RESULT_CM_ID varchar(19) primary key, - PATID number(38,0) NOT NULL, - ENCOUNTERID number(38,0) NULL, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, LAB_NAME varchar(10) NULL, SPECIMEN_SOURCE varchar(10) NULL, LAB_LOINC varchar(10) NULL, @@ -453,7 +453,7 @@ PMN_DROPSQL('DROP TABLE death'); END; / CREATE TABLE death( - PATID number(38,0) NOT NULL, + PATID varchar(50) NOT NULL, DEATH_DATE date NOT NULL, DEATH_DATE_IMPUTE varchar(2) NULL, DEATH_SOURCE varchar(2) NOT NULL, @@ -466,7 +466,7 @@ PMN_DROPSQL('DROP TABLE death_cause'); END; / CREATE TABLE death_cause( - PATID number(38,0) NOT NULL, + PATID varchar(50) NOT NULL, DEATH_CAUSE varchar(8) NOT NULL, DEATH_CAUSE_CODE varchar(2) NOT NULL, DEATH_CAUSE_TYPE varchar(2) NOT NULL, @@ -481,7 +481,7 @@ END; / CREATE TABLE dispensing( DISPENSINGID varchar(19) primary key, - PATID number(38,0) NOT NULL, + PATID varchar(50) NOT NULL, PRESCRIBINGID varchar(19) NULL, DISPENSE_DATE date NOT NULL, NDC varchar (11) NOT NULL, @@ -522,8 +522,8 @@ END; / CREATE TABLE prescribing( PRESCRIBINGID varchar(19) primary key, - PATID number(38,0) NOT NULL, - ENCOUNTERID number(38,0) NULL, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, RX_PROVIDERID varchar(50) NULL, -- NOTE: The spec has a _ before the ID, but this is inconsistent. RX_ORDER_DATE date NULL, RX_ORDER_TIME varchar (5) NULL, @@ -562,7 +562,7 @@ PMN_DROPSQL('DROP TABLE pcornet_trial'); END; / CREATE TABLE pcornet_trial( - PATID number(38,0) NOT NULL, + PATID varchar(50) NOT NULL, TRIALID varchar(20) NOT NULL, PARTICIPANTID varchar(50) NOT NULL, TRIAL_SITEID varchar(50) NULL, @@ -579,8 +579,8 @@ END; / CREATE TABLE condition( CONDITIONID varchar(19) primary key, - PATID number(38,0) NOT NULL, - ENCOUNTERID number(38,0) NULL, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, REPORT_DATE date NULL, RESOLVE_DATE date NULL, ONSET_DATE date NULL, @@ -618,8 +618,8 @@ END; / CREATE TABLE pro_cm( PRO_CM_ID varchar(19) primary key, - PATID number(38,0) NOT NULL, - ENCOUNTERID number(38,0) NULL, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, PRO_ITEM varchar (7) NOT NULL, PRO_LOINC varchar (10) NULL, PRO_DATE date NOT NULL, @@ -703,8 +703,8 @@ PMN_DROPSQL('DROP TABLE encounter'); END; / CREATE TABLE encounter( - PATID number(38,0) NOT NULL, - ENCOUNTERID number(38,0) NOT NULL, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NOT NULL, ADMIT_DATE date NULL, ADMIT_TIME varchar(5) NULL, DISCHARGE_DATE date NULL, @@ -732,7 +732,7 @@ PMN_DROPSQL('DROP TABLE demographic'); END; / CREATE TABLE demographic( - PATID number(38,0) NOT NULL, + PATID varchar(50) NOT NULL, BIRTH_DATE date NULL, BIRTH_TIME varchar(5) NULL, SEX varchar(2) NULL, From 722962c8e971dc90a4f3871fd82b26a1654bb77e Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 18 May 2017 09:59:20 -0500 Subject: [PATCH 256/507] Seperate transform procedure definition and transform prep into seperate files --- Oracle/PCORNetInit.sql | 738 +++++++++++++++++++++++++++++++++++ Oracle/PCORNetLoader_ora.sql | 738 ----------------------------------- 2 files changed, 738 insertions(+), 738 deletions(-) create mode 100644 Oracle/PCORNetInit.sql diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql new file mode 100644 index 0000000..24f847a --- /dev/null +++ b/Oracle/PCORNetInit.sql @@ -0,0 +1,738 @@ +create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS + BEGIN + DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. + tabname => table_name, + estimate_percent => 50, -- Percentage picked somewhat arbitrarily + cascade => TRUE, + degree => 16 + ); +END GATHER_TABLE_STATS; +/ + +create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS + BEGIN + EXECUTE IMMEDIATE sqlstring; + EXCEPTION + WHEN OTHERS THEN NULL; +END PMN_DROPSQL; +/ + +create or replace FUNCTION PMN_IFEXISTS(objnamestr VARCHAR2, objtypestr VARCHAR2) RETURN BOOLEAN AS +cnt NUMBER; +BEGIN + SELECT COUNT(*) + INTO cnt + FROM USER_OBJECTS + WHERE upper(OBJECT_NAME) = upper(objnamestr) + and upper(object_type) = upper(objtypestr); + + IF( cnt = 0 ) + THEN + --dbms_output.put_line('NO!'); + return FALSE; + ELSE + --dbms_output.put_line('YES!'); + return TRUE; + END IF; + +END PMN_IFEXISTS; +/ + + +create or replace PROCEDURE PMN_Execuatesql(sqlstring VARCHAR2) AS +BEGIN + EXECUTE IMMEDIATE sqlstring; + dbms_output.put_line(sqlstring); +END PMN_ExecuateSQL; +/ + +--ACK: http://dba.stackexchange.com/questions/9441/how-to-catch-and-handle-only-specific-oracle-exceptions +create or replace procedure create_error_table(table_name varchar2) as +sqltext varchar2(4000); + +begin + dbms_errlog.create_error_log(dml_table_name => table_name); +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -955 THEN + NULL; -- suppresses ORA-00955 exception ("name is already used by an existing object") + ELSE + RAISE; + END IF; +-- Delete rows from a previous run in case the table already existed +sqltext := 'delete from ERR$_' || table_name; +PMN_Execuatesql(sqltext); +end; +/ + + + + + + +CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT +/ + +CREATE OR REPLACE SYNONYM I2B2MEDFACT FOR OBSERVATION_FACT_MEDS +/ + +BEGIN +PMN_DROPSQL('DROP TABLE i2b2patient_list'); +END; +/ + +CREATE table i2b2patient_list as +select * from +( +select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') +) where ROWNUM<100000000 +/ + +create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) +/ + +create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. - tabname => table_name, - estimate_percent => 50, -- Percentage picked somewhat arbitrarily - cascade => TRUE, - degree => 16 - ); -END GATHER_TABLE_STATS; -/ - -create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS - BEGIN - EXECUTE IMMEDIATE sqlstring; - EXCEPTION - WHEN OTHERS THEN NULL; -END PMN_DROPSQL; -/ - -create or replace FUNCTION PMN_IFEXISTS(objnamestr VARCHAR2, objtypestr VARCHAR2) RETURN BOOLEAN AS -cnt NUMBER; -BEGIN - SELECT COUNT(*) - INTO cnt - FROM USER_OBJECTS - WHERE upper(OBJECT_NAME) = upper(objnamestr) - and upper(object_type) = upper(objtypestr); - - IF( cnt = 0 ) - THEN - --dbms_output.put_line('NO!'); - return FALSE; - ELSE - --dbms_output.put_line('YES!'); - return TRUE; - END IF; - -END PMN_IFEXISTS; -/ - - -create or replace PROCEDURE PMN_Execuatesql(sqlstring VARCHAR2) AS -BEGIN - EXECUTE IMMEDIATE sqlstring; - dbms_output.put_line(sqlstring); -END PMN_ExecuateSQL; -/ - ---ACK: http://dba.stackexchange.com/questions/9441/how-to-catch-and-handle-only-specific-oracle-exceptions -create or replace procedure create_error_table(table_name varchar2) as -sqltext varchar2(4000); - -begin - dbms_errlog.create_error_log(dml_table_name => table_name); -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE = -955 THEN - NULL; -- suppresses ORA-00955 exception ("name is already used by an existing object") - ELSE - RAISE; - END IF; --- Delete rows from a previous run in case the table already existed -sqltext := 'delete from ERR$_' || table_name; -PMN_Execuatesql(sqltext); -end; -/ - - - - - - -CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT -/ - -CREATE OR REPLACE SYNONYM I2B2MEDFACT FOR OBSERVATION_FACT_MEDS -/ - -BEGIN -PMN_DROPSQL('DROP TABLE i2b2patient_list'); -END; -/ - -CREATE table i2b2patient_list as -select * from -( -select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') -) where ROWNUM<100000000 -/ - -create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) -/ - -create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE Date: Thu, 18 May 2017 10:01:46 -0500 Subject: [PATCH 257/507] Updated PCORNetLoader procedure with param used to declare starting point --- Oracle/PCORNetLoader_ora.sql | 122 ++++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 37 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index e75d011..52717df 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1210,44 +1210,92 @@ end pcornetReport; -create or replace procedure pcornetloader as +create or replace PROCEDURE PCORNetLoader(start_with VARCHAR2) AS begin ----pcornetclear; -PCORNetDemographic; -PCORNetEncounter; -PCORNetDiagnosis; -PCORNetCondition; -PCORNetProcedure; -PCORNetVital; -PCORNetEnroll; -PCORNetLabResultCM; -PCORNetPrescribing; - -/* ORA-04068: existing state of packages has been discarded -ORA-04065: not executed, altered or dropped stored procedure "PCORNETDISPENSING" -ORA-06508: PL/SQL: could not find program unit being called: "PCORNETDISPENSING" -ORA-06512: at "PCORNETLOADER", line 14 -ORA-06512: at line 2 -04068. 00000 - "existing state of packages%s%s%s has been discarded" -*Cause: One of errors 4060 - 4067 when attempt to execute a stored - procedure. -*Action: Try again after proper re-initialization of any application's - state. - -The above error only happens when we call PCORNetDispensing _and_ PCORNetPrescribing -from within pcornetloader. When running either individually, the error does not -happen. - -Skipping dispensing as per gpc-dev notes: -http://listserv.kumc.edu/pipermail/gpc-dev/attachments/20160223/8d79fa70/attachment-0001.pdf -> LV: the dispensing side [?] is not mandatory? we just did Rx, since that -> what we have in our i2b2 -*/ -PCORNetDispensing; -PCORNetDeath; -PCORNetHarvest; - -end pcornetloader; + if start_with in ('PCORNetDemographic') then + PCORNetDemographic; + end if; + + if start_with in ('PCORNetDemographic', 'PCORNetEncounter') then + PCORNetEncounter; + end if; + + if start_with in ( + 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis' + ) then + PCORNetDiagnosis; + end if; + + if start_with in ( + 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', + 'PCORNetCondition' + ) then + PCORNetCondition; + end if; + + if start_with in ( + 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', + 'PCORNetCondition', 'PCORNetProcedure' + ) then + PCORNetProcedure; + end if; + + if start_with in ( + 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', + 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital' + ) then + PCORNetVital; + end if; + + if start_with in ( + 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', + 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll' + ) then + PCORNetEnroll; + end if; + + if start_with in ( + 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', + 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', + 'PCORNetLabResultCM' + ) then + PCORNetLabResultCM; + end if; + + if start_with in ( + 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', + 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', + 'PCORNetLabResultCM', 'PCORNetPrescribing' + ) then + PCORNetPrescribing; + end if; + + if start_with in ( + 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', + 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', + 'PCORNetLabResultCM', 'PCORNetPrescribing', 'PCORNetDispensing' + ) then + PCORNetDispensing; + end if; + + if start_with in ( + 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', + 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', + 'PCORNetLabResultCM', 'PCORNetPrescribing', 'PCORNetDispensing', + 'PCORNetDeath' + ) then + PCORNetDeath; + end if; + + if start_with in ( + 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', + 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', + 'PCORNetLabResultCM', 'PCORNetPrescribing', 'PCORNetDispensing', + 'PCORNetDeath', 'PCORNetHarvest' + ) then + PCORNetHarvest; + end if; +end PCORNetLoader; / From d4aa672acbba11b8e0e793ee39d98e173e884d84 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 18 May 2017 10:03:32 -0500 Subject: [PATCH 258/507] Updated run shell script to reflect changes to PCORNetLoader and PCORNetInit --- Oracle/run-i2p-transform.sh | 96 +++++++++++++++++++++++++++---------- 1 file changed, 72 insertions(+), 24 deletions(-) diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index ac7967e..960dff2 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -1,28 +1,6 @@ #!/bin/bash set -e -# Expected environment variables (put there by Jenkins, etc.) - -# Database SID -#export sid= -# User and password for CDM user -#export pcornet_cdm_user= -#export pcornet_cdm= - -#export i2b2_data_schema= -#export i2b2_meta_schema= -#export datamart_id= -#export datamart_name= -#export network_id= -#export network_name= -# export i2b2_etl_schema= -# export min_pat_list_date_dd_mon_rrrr= -# export min_visit_date_dd_mon_rrrr= -# export enrollment_months_back=36 - -# All i2b2 terms - used for local path mapping -#export terms_table= - python load_csv.py harvest_local harvest_local.csv harvest_local.ctl pcornet_cdm_user pcornet_cdm python load_csv.py PMN_LabNormal pmn_labnormal.csv pmn_labnormal.ctl pcornet_cdm_user pcornet_cdm python load_csv.py lab_loinc_mapping lab_loinc_mapping.csv lab_loinc_mapping.ctl pcornet_cdm_user pcornet_cdm @@ -58,11 +36,46 @@ select rxcui from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id where 1=0; select case when qty > 0 then 1 else 1/0 end obs_fact_meds_populated from ( select count(*) qty from observation_fact_meds ); +EOF + + +########### Initialize Schema ########### +if [ ${initialize_schema} == 'true'] +then + +sqlplus /nolog < Date: Thu, 18 May 2017 13:11:42 -0500 Subject: [PATCH 259/507] A couple bug fixes --- Oracle/PCORNetLoader_ora.sql | 11 ----------- Oracle/run-i2p-transform.sh | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 52717df..5d8fc75 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1297,14 +1297,3 @@ begin end if; end PCORNetLoader; / - - -BEGIN -pcornetloader; --- you may want to run sql statements one by one in the pcornetloader procedure :) -END; -/ - -select concept "Data Type",sourceval "From i2b2",destval "In PopMedNet", diff "Difference" from i2preport where RUNID = (select max(RUNID) from I2PREPORT); - - - diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 960dff2..2032aec 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -40,7 +40,7 @@ EOF ########### Initialize Schema ########### -if [ ${initialize_schema} == 'true'] +if [ ${initialize_schema} == 'true' ] then sqlplus /nolog < Date: Tue, 23 May 2017 14:12:45 -0500 Subject: [PATCH 260/507] Added TRUNCATE TABLE statements to make transform procedures idempotent --- Oracle/PCORNetLoader_ora.sql | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 5d8fc75..969f703 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -24,6 +24,8 @@ create or replace procedure PCORNetDemographic as +execute immediate 'truncate table demographic'; + sqltext varchar2(4000); cursor getsql is --1 -- S,R,NH @@ -193,6 +195,8 @@ begin PMN_DROPSQL('drop index encounter_patid'); PMN_DROPSQL('drop index encounter_encounterid'); +execute immediate 'truncate table encounter'; + insert into encounter(PATID,ENCOUNTERID,admit_date ,ADMIT_TIME , DISCHARGE_DATE ,DISCHARGE_TIME ,PROVIDERID ,FACILITY_LOCATION ,ENC_TYPE ,FACILITYID ,DISCHARGE_DISPOSITION , @@ -244,6 +248,8 @@ begin PMN_DROPSQL('drop index diagnosis_patid'); PMN_DROPSQL('drop index diagnosis_encounterid'); +execute immediate 'truncate table diagnosis'; + PMN_DROPSQL('DROP TABLE sourcefact'); -- associated indexes will be dropped as well sqltext := 'create table sourcefact as '|| @@ -355,6 +361,8 @@ begin PMN_DROPSQL('drop index condition_patid'); PMN_DROPSQL('drop index condition_encounterid'); +execute immediate 'truncate table condition'; + PMN_DROPSQL('DROP TABLE sourcefact2'); sqltext := 'create table sourcefact2 as '|| @@ -407,6 +415,8 @@ begin PMN_DROPSQL('drop index procedures_patid'); PMN_DROPSQL('drop index procedures_encounterid'); +execute immediate 'truncate table procedures'; + insert into procedures( patid, encounterid, enc_type, admit_date, px_date, providerid, px, px_type, px_source) select distinct fact.patient_num, enc.encounterid, enc.enc_type, enc.admit_date, fact.start_date, @@ -437,6 +447,8 @@ begin PMN_DROPSQL('drop index vital_patid'); PMN_DROPSQL('drop index vital_encounterid'); +execute immediate 'truncate table vital'; + -- jgk: I took out admit_date - it doesn't appear in the scheme. Now in SQLServer format - date, substring, name on inner select, no nested with. Added modifiers and now use only pathnames, not codes. insert into vital(patid, encounterid, measure_date, measure_time,vital_source,ht, wt, diastolic, systolic, original_bmi, bp_position,smoking,tobacco,tobacco_type) select patid, encounterid, to_date(measure_date,'rrrr-mm-dd') measure_date, measure_time,vital_source,ht, wt, diastolic, systolic, original_bmi, bp_position,smoking,tobacco, @@ -527,6 +539,8 @@ begin PMN_DROPSQL('drop index enrollment_patid'); +execute immediate 'truncate table enrollment'; + INSERT INTO enrollment(PATID, ENR_START_DATE, ENR_END_DATE, CHART, ENR_BASIS) with pats_delta as ( -- If only one visit, visit_delta_days will be 0 @@ -598,6 +612,8 @@ begin PMN_DROPSQL('drop index lab_result_cm_patid'); PMN_DROPSQL('drop index lab_result_cm_encounterid'); +execute immediate 'truncate table lab_result_cm'; + PMN_DROPSQL('DROP TABLE priority'); sqltext := 'create table priority as '|| @@ -734,6 +750,8 @@ END PCORNetLabResultCM; create or replace procedure PCORNetHarvest as begin +execute immediate 'truncate table harvest'; + INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.PX_DATE_MGMT, hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, case when (select count(*) from demographic) > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, @@ -830,6 +848,7 @@ begin PMN_DROPSQL('drop index prescribing_patid'); PMN_DROPSQL('drop index prescribing_encounterid'); +execute immediate 'truncate table prescribing'; PMN_DROPSQL('DROP TABLE basis'); sqltext := 'create table basis as '|| @@ -989,9 +1008,11 @@ whenever sqlerror exit; create or replace procedure PCORNetDispensing as sqltext varchar2(4000); begin -/* + PMN_DROPSQL('drop index dispensing_patid'); +execute immediate 'truncate table dispensing'; +/* PMN_DROPSQL('DROP TABLE supply'); sqltext := 'create table supply as '|| '(select nval_num,encounter_num,concept_cd from i2b2fact supply '|| @@ -1118,6 +1139,8 @@ end PCORNetDispensing; create or replace procedure PCORNetDeath as begin + +execute immediate 'truncate table death'; insert into death( patid, death_date, death_date_impute, death_source, death_match_confidence) select distinct pat.patient_num, pat.death_date, From 1619102f53f7266757bbdc2f09d4bdb5073dd769 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 23 May 2017 14:53:17 -0500 Subject: [PATCH 261/507] Added intermediate table for DRGs in ENCOUNTER transform in attempt to improve performance. --- Oracle/PCORNetLoader_ora.sql | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 969f703..13f214d 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -197,6 +197,19 @@ PMN_DROPSQL('drop index encounter_encounterid'); execute immediate 'truncate table encounter'; +PMN_DROPSQL('DROP TABLE drg'); -- associated indexes will be dropped as well + +sqltext := 'create table drg as '|| + 'select * from' || + 'select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from '|| + '(select patient_num,encounter_num,drg_type,max(drg) drg from '|| + '(select distinct f.patient_num,encounter_num,SUBSTR(c_fullname,22,2) drg_type,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,3) drg from i2b2fact f '|| + 'inner join demographic d on f.patient_num=d.patid '|| + 'inner join pcornet_enc enc on enc.c_basecode = f.concept_cd '|| + ' and enc.c_fullname like ''\PCORI\ENCOUNTER\DRG\%'') drg1 group by patient_num,encounter_num,drg_type) drg '|| + 'where rn=1'; +PMN_EXECUATESQL(sqltext); + insert into encounter(PATID,ENCOUNTERID,admit_date ,ADMIT_TIME , DISCHARGE_DATE ,DISCHARGE_TIME ,PROVIDERID ,FACILITY_LOCATION ,ENC_TYPE ,FACILITYID ,DISCHARGE_DISPOSITION , @@ -215,15 +228,7 @@ select distinct v.patient_num, v.encounter_num, drg.drg, drg_type, CASE WHEN admitting_source IS NULL THEN 'NI' ELSE admitting_source END admitting_source from i2b2visit v inner join demographic d on v.patient_num=d.patid -left outer join - (select * from - (select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from - (select patient_num,encounter_num,drg_type,max(drg) drg from - (select distinct f.patient_num,encounter_num,SUBSTR(c_fullname,22,2) drg_type,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,3) drg from i2b2fact f - inner join demographic d on f.patient_num=d.patid - inner join pcornet_enc enc on enc.c_basecode = f.concept_cd - and enc.c_fullname like '\PCORI\ENCOUNTER\DRG\%') drg1 group by patient_num,encounter_num,drg_type) drg) drg - where rn=1) drg -- This section is bugfixed to only include 1 drg if multiple DRG types exist in a single encounter... +left outer join drg -- This section is bugfixed to only include 1 drg if multiple DRG types exist in a single encounter... on drg.patient_num=v.patient_num and drg.encounter_num=v.encounter_num left outer join -- Encounter type. Note that this requires a full table scan on the ontology table, so it is not particularly efficient. From 212c86cf3e05a83cd19f0a5ddf26535359ee929e Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 24 May 2017 08:31:48 -0500 Subject: [PATCH 262/507] Fixed two small errors in transform procedures. --- Oracle/PCORNetLoader_ora.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 13f214d..fbacb45 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -24,8 +24,6 @@ create or replace procedure PCORNetDemographic as -execute immediate 'truncate table demographic'; - sqltext varchar2(4000); cursor getsql is --1 -- S,R,NH @@ -162,6 +160,8 @@ pcornet_popcodelist; PMN_DROPSQL('drop index demographic_patid'); +execute immediate 'truncate table demographic'; + OPEN getsql; LOOP FETCH getsql INTO sqltext; @@ -203,7 +203,7 @@ sqltext := 'create table drg as '|| 'select * from' || 'select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from '|| '(select patient_num,encounter_num,drg_type,max(drg) drg from '|| - '(select distinct f.patient_num,encounter_num,SUBSTR(c_fullname,22,2) drg_type,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,3) drg from i2b2fact f '|| + '(select distinct f.patient_num,encounter_num,SUBSTR(c_fullname,22,2) drg_type,SUBSTR(pcori_basecode,INSTR(pcori_basecode, '':'')+1,3) drg from i2b2fact f '|| 'inner join demographic d on f.patient_num=d.patid '|| 'inner join pcornet_enc enc on enc.c_basecode = f.concept_cd '|| ' and enc.c_fullname like ''\PCORI\ENCOUNTER\DRG\%'') drg1 group by patient_num,encounter_num,drg_type) drg '|| From ecce0dc506c91f6af0028e6ac9a06f529915b299 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 24 May 2017 09:08:49 -0500 Subject: [PATCH 263/507] Fix typo in PCORNetEncounter DRG intermediary table creation --- Oracle/PCORNetLoader_ora.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index fbacb45..64bdb57 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -200,13 +200,13 @@ execute immediate 'truncate table encounter'; PMN_DROPSQL('DROP TABLE drg'); -- associated indexes will be dropped as well sqltext := 'create table drg as '|| - 'select * from' || - 'select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from '|| + 'select * from '|| + '(select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from '|| '(select patient_num,encounter_num,drg_type,max(drg) drg from '|| - '(select distinct f.patient_num,encounter_num,SUBSTR(c_fullname,22,2) drg_type,SUBSTR(pcori_basecode,INSTR(pcori_basecode, '':'')+1,3) drg from i2b2fact f '|| + '(select distinct f.patient_num,encounter_num,SUBSTR(c_fullname,22,2) drg_type,SUBSTR(pcori_basecode,INSTR(pcori_basecode, '':'')+1,3) drg from i2b2fact f '|| 'inner join demographic d on f.patient_num=d.patid '|| 'inner join pcornet_enc enc on enc.c_basecode = f.concept_cd '|| - ' and enc.c_fullname like ''\PCORI\ENCOUNTER\DRG\%'') drg1 group by patient_num,encounter_num,drg_type) drg '|| + 'and enc.c_fullname like ''\PCORI\ENCOUNTER\DRG\%'') drg1 group by patient_num,encounter_num,drg_type) drg) drg '|| 'where rn=1'; PMN_EXECUATESQL(sqltext); From a283279feee884a9372775451f7766f51f1d1210 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 24 May 2017 10:34:12 -0500 Subject: [PATCH 264/507] Moved creation of intermediary tables to PCORNetInit.sql --- Oracle/PCORNetInit.sql | 130 ++++++++++++++++++++++++++++++++++- Oracle/PCORNetLoader_ora.sql | 122 -------------------------------- 2 files changed, 129 insertions(+), 123 deletions(-) diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index 24f847a..92d554e 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -735,4 +735,132 @@ BEGIN insert into i2preport (runid) values (0); pcornet_popcodelist; END; -/ \ No newline at end of file +/ + + +/* TODO: When compiling PCORNetLabResultCM I got Error(106,17): + PL/SQL: ORA-00942: table or view does not exist +apparently due to tables that the procedure references but also drops/recreates +before reference. Creating them outside the function solves the issue. SQL +copied from the function. + +In the same function, I got + Error(63,123): PL/SQL: ORA-00904: "LAB"."PCORI_SPECIMEN_SOURCE": invalid identifier +So, I just altered the table to have the referenced column. + +*/ +whenever sqlerror continue; +drop table priority; +drop table location; + +create table priority ( + patient_num number(38,0), + encounter_num number(38,0), + provider_id varchar2(50 byte), + concept_cd varchar2(50 byte), + start_date date, + priority varchar2(50 byte) + ); + +create table location ( + patient_num number(38,0), + encounter_num number(38,0), + provider_id varchar2(50 byte), + concept_cd varchar2(50 byte), + start_date date, + result_loc varchar2(50 byte) + ); + +alter table "&&i2b2_meta_schema".pcornet_lab add ( + pcori_specimen_source varchar2(1000) -- arbitrary + ); +whenever sqlerror exit; + + +/* TODO: When compiling PCORNetPrescribing, I got Error(93,15): + PL/SQL: ORA-00942: table or view does not exist +At compile time, it's complaining about the fact tables don't exist that are +created in the function itself. I created them ahead of time - SQL taken from +the procedure. +*/ +whenever sqlerror continue; +drop table basis; +drop table freq; +drop table quantity; +drop table refills; +drop table supply; + +create table basis ( + pcori_basecode varchar2(50 byte), + c_fullname varchar2(700 byte), + encounter_num number(38,0), + concept_cd varchar2(50 byte), + instance_num number(18,0), + start_date date, + provider_id varchar2(50 byte), + modifier_cd varchar2(100 byte) + ) ; + +create table freq ( + pcori_basecode varchar2(50 byte), + encounter_num number(38,0), + concept_cd varchar2(50 byte), + instance_num number(18,0), + start_date date, + provider_id varchar2(50 byte), + modifier_cd varchar2(100 byte) + ); + +create table quantity( + nval_num number(18,5), + encounter_num number(38,0), + concept_cd varchar2(50 byte), + instance_num number(18,0), + start_date date, + provider_id varchar2(50 byte), + modifier_cd varchar2(100 byte) + ); + +create table refills( + nval_num number(18,5), + encounter_num number(38,0), + concept_cd varchar2(50 byte), + instance_num number(18,0), + start_date date, + provider_id varchar2(50 byte), + modifier_cd varchar2(100 byte) + ); + +create table supply( + nval_num number(18,5), + encounter_num number(38,0), + concept_cd varchar2(50 byte), + instance_num number(18,0), + start_date date, + provider_id varchar2(50 byte), + modifier_cd varchar2(100 byte) + ); + +whenever sqlerror exit; + + +/* TODO: When compiling PCORNetDispensing: + +Error(53,16): PL/SQL: ORA-00942: table or view does not exist (amount) + - supply, also used, is created above in the prescribing function + +Also, Error(57,57): PL/SQL: ORA-00904: "MO"."PCORI_NDC": invalid identifier +*/ +whenever sqlerror continue; +drop table amount; + +create table amount( + nval_num number(18,5), + encounter_num number(38,0), + concept_cd varchar2(50 byte) + ); + +alter table "&&i2b2_meta_schema".pcornet_med add ( + pcori_ndc varchar2(1000) -- arbitrary + ); +whenever sqlerror exit; \ No newline at end of file diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 64bdb57..2a9f825 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -572,43 +572,6 @@ GATHER_TABLE_STATS('ENROLLMENT'); end PCORNetEnroll; / -/* TODO: When compiling PCORNetLabResultCM I got Error(106,17): - PL/SQL: ORA-00942: table or view does not exist -apparently due to tables that the procedure references but also drops/recreates -before reference. Creating them outside the function solves the issue. SQL -copied from the function. - -In the same function, I got - Error(63,123): PL/SQL: ORA-00904: "LAB"."PCORI_SPECIMEN_SOURCE": invalid identifier -So, I just altered the table to have the referenced column. - -*/ -whenever sqlerror continue; -drop table priority; -drop table location; - -create table priority ( - patient_num number(38,0), - encounter_num number(38,0), - provider_id varchar2(50 byte), - concept_cd varchar2(50 byte), - start_date date, - priority varchar2(50 byte) - ); - -create table location ( - patient_num number(38,0), - encounter_num number(38,0), - provider_id varchar2(50 byte), - concept_cd varchar2(50 byte), - start_date date, - result_loc varchar2(50 byte) - ); - -alter table "&&i2b2_meta_schema".pcornet_lab add ( - pcori_specimen_source varchar2(1000) -- arbitrary - ); -whenever sqlerror exit; create or replace procedure PCORNetLabResultCM as sqltext varchar2(4000); @@ -780,71 +743,6 @@ end PCORNetHarvest; -/* TODO: When compiling PCORNetPrescribing, I got Error(93,15): - PL/SQL: ORA-00942: table or view does not exist -At compile time, it's complaining about the fact tables don't exist that are -created in the function itself. I created them ahead of time - SQL taken from -the procedure. -*/ -whenever sqlerror continue; -drop table basis; -drop table freq; -drop table quantity; -drop table refills; -drop table supply; - -create table basis ( - pcori_basecode varchar2(50 byte), - c_fullname varchar2(700 byte), - encounter_num number(38,0), - concept_cd varchar2(50 byte), - instance_num number(18,0), - start_date date, - provider_id varchar2(50 byte), - modifier_cd varchar2(100 byte) - ) ; - -create table freq ( - pcori_basecode varchar2(50 byte), - encounter_num number(38,0), - concept_cd varchar2(50 byte), - instance_num number(18,0), - start_date date, - provider_id varchar2(50 byte), - modifier_cd varchar2(100 byte) - ); - -create table quantity( - nval_num number(18,5), - encounter_num number(38,0), - concept_cd varchar2(50 byte), - instance_num number(18,0), - start_date date, - provider_id varchar2(50 byte), - modifier_cd varchar2(100 byte) - ); - -create table refills( - nval_num number(18,5), - encounter_num number(38,0), - concept_cd varchar2(50 byte), - instance_num number(18,0), - start_date date, - provider_id varchar2(50 byte), - modifier_cd varchar2(100 byte) - ); - -create table supply( - nval_num number(18,5), - encounter_num number(38,0), - concept_cd varchar2(50 byte), - instance_num number(18,0), - start_date date, - provider_id varchar2(50 byte), - modifier_cd varchar2(100 byte) - ); - -whenever sqlerror exit; create or replace procedure PCORNetPrescribing as sqltext varchar2(4000); @@ -989,26 +887,6 @@ end PCORNetPrescribing; -/* TODO: When compiling PCORNetDispensing: - -Error(53,16): PL/SQL: ORA-00942: table or view does not exist (amount) - - supply, also used, is created above in the prescribing function - -Also, Error(57,57): PL/SQL: ORA-00904: "MO"."PCORI_NDC": invalid identifier -*/ -whenever sqlerror continue; -drop table amount; - -create table amount( - nval_num number(18,5), - encounter_num number(38,0), - concept_cd varchar2(50 byte) - ); - -alter table "&&i2b2_meta_schema".pcornet_med add ( - pcori_ndc varchar2(1000) -- arbitrary - ); -whenever sqlerror exit; create or replace procedure PCORNetDispensing as sqltext varchar2(4000); From 89aba652bf45b9da18ac3a00d425ac66e3a9b8b5 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 25 May 2017 10:09:26 -0500 Subject: [PATCH 265/507] SQL to create logging table, task sequence, and procedures for manipulation --- Oracle/PCORNetLogging.sql | 54 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 Oracle/PCORNetLogging.sql diff --git a/Oracle/PCORNetLogging.sql b/Oracle/PCORNetLogging.sql new file mode 100644 index 0000000..70abadf --- /dev/null +++ b/Oracle/PCORNetLogging.sql @@ -0,0 +1,54 @@ +-------------------------------------------------------------------------------- +-- PCORNetLogging Script +-- +-- This file contains the Oracle SQL needed to create the logging table and +-- associated sequence. +-------------------------------------------------------------------------------- + + +whenever sqlerror continue; +drop table i2p_task_log; +whenever sqlerror exit; + +create table i2p_task_log ( + task_num number(8) not null, + cdm_release varchar(48) not null, + task_name varchar(48) not null, + build_num number(8) not null, + started date not null, + completed date, + num_rows number(16), + data_source varchar(48) +); + + +whenever sqlerror continue; +drop sequence i2p_task_seq; +whenever sqlerror exit; + +create sequence i2p_task_seq + minvalue 1 + start with 1 + increment by 1 + nocache +; + + +create or replace procedure LogTaskStart(r_name varchar2, t_name varchar2, b_num number, d_source varchar2) as +begin + insert into i2p_task_log (task_num, release_name, task_name, build_num, started, data_source) + values (i2p_task_seq.nextval, r_name, t_name, b_num, sysdate, d_source); +end LogTaskStart; + + +create or replace procedure LogTaskComplete(r_name varchar2, t_name varchar2, b_num number, target_table varchar2) as +begin + execute immediate + 'update i2p_task_log set (completed, num_rows) = (' || + ' select sysdate, count(*) from ' || target_table || + ' )' || + 'where release_name=''' || r_name || '''' || + ' and task_name=''' || t_name || '''' || + ' and build_num=''' || b_num || '''' + ; +end LogTaskComplete; \ No newline at end of file From 573eb6c985c39b2c41600d535bd2ccf0d269eba9 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 25 May 2017 10:10:12 -0500 Subject: [PATCH 266/507] Updated transform to use logging transform. --- Oracle/PCORNetLoader_ora.sql | 28 +++++++++++++++++++++++++++- Oracle/run-i2p-transform.sh | 2 +- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 2a9f825..88b360b 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1116,48 +1116,64 @@ end pcornetReport; -create or replace PROCEDURE PCORNetLoader(start_with VARCHAR2) AS +create or replace PROCEDURE PCORNetLoader(start_with VARCHAR2, + release_name VARCHAR2, build_num NUMBER, data_source VARCHAR2 +) AS begin if start_with in ('PCORNetDemographic') then + LogTaskStart(release_name, 'PCORNetDemographic', build_num, data_source); PCORNetDemographic; + LogTaskComplete(release_name, 'PCORNetDemographic', build_num, 'DEMOGRAPHIC'); end if; if start_with in ('PCORNetDemographic', 'PCORNetEncounter') then + LogTaskStart(release_name, 'PCORNetEncounter', build_num, data_source); PCORNetEncounter; + LogTaskComplete(release_name, 'PCORNetEncounter', build_num, data_source); end if; if start_with in ( 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis' ) then + LogTaskStart(release_name, 'PCORNetDiagnosis', build_num, data_source); PCORNetDiagnosis; + LogTaskComplete(release_name, 'PCORNetDiagnosis', build_num, data_source); end if; if start_with in ( 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', 'PCORNetCondition' ) then + LogTaskStart(release_name, 'PCORNetCondition', build_num, data_source); PCORNetCondition; + LogTaskComplete(release_name, 'PCORNetCondition', build_num, data_source); end if; if start_with in ( 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', 'PCORNetCondition', 'PCORNetProcedure' ) then + LogTaskStart(release_name, 'PCORNetProcedure', build_num, data_source); PCORNetProcedure; + LogTaskComplete(release_name, 'PCORNetProcedure', build_num, data_source); end if; if start_with in ( 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital' ) then + LogTaskStart(release_name, 'PCORNetVital', build_num, data_source); PCORNetVital; + LogTaskComplete(release_name, 'PCORNetVital', build_num, data_source); end if; if start_with in ( 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll' ) then + LogTaskStart(release_name, 'PCORNetEnroll', build_num, data_source); PCORNetEnroll; + LogTaskComplete(release_name, 'PCORNetEnroll', build_num, data_source); end if; if start_with in ( @@ -1165,7 +1181,9 @@ begin 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', 'PCORNetLabResultCM' ) then + LogTaskStart(release_name, 'PCORNetLabResultCM', build_num, data_source); PCORNetLabResultCM; + LogTaskComplete(release_name, 'PCORNetLabResultCM', build_num, data_source); end if; if start_with in ( @@ -1173,7 +1191,9 @@ begin 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', 'PCORNetLabResultCM', 'PCORNetPrescribing' ) then + LogTaskStart(release_name, 'PCORNetPrescribing', build_num, data_source); PCORNetPrescribing; + LogTaskComplete(release_name, 'PCORNetPrescribing', build_num, data_source); end if; if start_with in ( @@ -1181,7 +1201,9 @@ begin 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', 'PCORNetLabResultCM', 'PCORNetPrescribing', 'PCORNetDispensing' ) then + LogTaskStart(release_name, 'PCORNetDispensing', build_num, data_source); PCORNetDispensing; + LogTaskComplete(release_name, 'PCORNetDispensing', build_num, data_source); end if; if start_with in ( @@ -1190,7 +1212,9 @@ begin 'PCORNetLabResultCM', 'PCORNetPrescribing', 'PCORNetDispensing', 'PCORNetDeath' ) then + LogTaskStart(release_name, 'PCORNetDeath', build_num, data_source); PCORNetDeath; + LogTaskComplete(release_name, 'PCORNetDeath', build_num, data_source); end if; if start_with in ( @@ -1199,7 +1223,9 @@ begin 'PCORNetLabResultCM', 'PCORNetPrescribing', 'PCORNetDispensing', 'PCORNetDeath', 'PCORNetHarvest' ) then + LogTaskStart(release_name, 'PCORNetHarvest', build_num, data_source); PCORNetHarvest; + LogTaskComplete(release_name, 'PCORNetHarvest', build_num, data_source); end if; end PCORNetLoader; / diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 2032aec..49afe0d 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -117,7 +117,7 @@ set linesize 3000; set pagesize 5000; -- Run transform -execute PCORNetLoader('${start_with}'); +execute PCORNetLoader('${start_with}', '${release_name}', '${build_num}', $'{data_source}'); EOF fi From 0d47d973f47133ffa8da9c451f2d02e7f73027c0 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 25 May 2017 13:28:02 -0500 Subject: [PATCH 267/507] Handful of small fixes --- Oracle/PCORNetLogging.sql | 10 ++++++---- Oracle/run-i2p-transform.sh | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Oracle/PCORNetLogging.sql b/Oracle/PCORNetLogging.sql index 70abadf..c4d13a1 100644 --- a/Oracle/PCORNetLogging.sql +++ b/Oracle/PCORNetLogging.sql @@ -1,8 +1,8 @@ -------------------------------------------------------------------------------- -- PCORNetLogging Script -- --- This file contains the Oracle SQL needed to create the logging table and --- associated sequence. +-- This file contains the Oracle SQL needed to create the logging table, the +-- associated sequence, and procedures for manipulating the log table. -------------------------------------------------------------------------------- @@ -12,7 +12,7 @@ whenever sqlerror exit; create table i2p_task_log ( task_num number(8) not null, - cdm_release varchar(48) not null, + release_name varchar(48) not null, task_name varchar(48) not null, build_num number(8) not null, started date not null, @@ -39,6 +39,7 @@ begin insert into i2p_task_log (task_num, release_name, task_name, build_num, started, data_source) values (i2p_task_seq.nextval, r_name, t_name, b_num, sysdate, d_source); end LogTaskStart; +/ create or replace procedure LogTaskComplete(r_name varchar2, t_name varchar2, b_num number, target_table varchar2) as @@ -51,4 +52,5 @@ begin ' and task_name=''' || t_name || '''' || ' and build_num=''' || b_num || '''' ; -end LogTaskComplete; \ No newline at end of file +end LogTaskComplete; +/ \ No newline at end of file diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 49afe0d..bd95c05 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -117,7 +117,7 @@ set linesize 3000; set pagesize 5000; -- Run transform -execute PCORNetLoader('${start_with}', '${release_name}', '${build_num}', $'{data_source}'); +execute PCORNetLoader('${start_with}', '${release_name}', '${BUILD_NUMBER}', $'{data_source}'); EOF fi From 2ece65f423da5ffab3abf38f173a9b50278508be Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 25 May 2017 15:45:46 -0500 Subject: [PATCH 268/507] Misplaced quote --- Oracle/run-i2p-transform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index bd95c05..d896fd9 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -117,7 +117,7 @@ set linesize 3000; set pagesize 5000; -- Run transform -execute PCORNetLoader('${start_with}', '${release_name}', '${BUILD_NUMBER}', $'{data_source}'); +execute PCORNetLoader('${start_with}', '${release_name}', '${BUILD_NUMBER}', '${data_source}'); EOF fi From 2dc151b36203404fbc3743229bd2f4b2d80c5225 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 1 Jun 2017 09:57:45 -0500 Subject: [PATCH 269/507] Moved post proc tasks into a procedure to better control execution flow. --- Oracle/PCORNetLoader_ora.sql | 51 +++++++++++++++++++++++++++++++++++- Oracle/cdm_postproc.sql | 37 -------------------------- Oracle/run-i2p-transform.sh | 5 +--- 3 files changed, 51 insertions(+), 42 deletions(-) delete mode 100644 Oracle/cdm_postproc.sql diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 2a9f825..782d0fa 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1113,7 +1113,54 @@ insert into i2pReport values(v_runid, SYSDATE, 'Dispensing', null, dispensings, end pcornetReport; / - +-------------------------------------------------------------------------------- +-- PCORNetPostProc procedure +-- +-- Ideally this procedure would be empty, but as a matter of practice there are +-- often post processing cleanup tasks that needs to be complete. Such tasks +-- should reside here with the goal of incorporating them into to the proper +-- procedures above. +-------------------------------------------------------------------------------- +create or replace procedure PCORNetPostProc as +begin + + /* Copy providerid from encounter table to diagnosis, procedures tables. + CDM specification says: + "Please note: This is a field replicated from the ENCOUNTER table." + */ + merge into diagnosis d + using encounter e + on (d.encounterid = e.encounterid) + when matched then update set d.providerid = e.providerid; + + merge into procedures p + using encounter e + on (p.encounterid = e.encounterid) + when matched then update set p.providerid = e.providerid; + + merge into prescribing p + using encounter e + on (p.encounterid = e.encounterid) + when matched then update set p.rx_providerid = e.providerid; + + /* Currently in HERON, we have hight in cm and weight in oz (from visit vitals). + The CDM wants height in inches and weight in pounds. */ + update vital v set v.ht = v.ht / 2.54; + update vital v set v.wt = v.wt / 16; + + /* Remove rows from the PRESCRIBING table where RX_* fields are null + TODO: Remove this when fixed in HERON + */ + delete + from prescribing + where rx_basis is null + and rx_quantity is null + and rx_frequency is null + and rx_refills is null + ; + +end PCORNetPostProc; +/ create or replace PROCEDURE PCORNetLoader(start_with VARCHAR2) AS @@ -1201,5 +1248,7 @@ begin ) then PCORNetHarvest; end if; + + PCORNetPostProc; end PCORNetLoader; / diff --git a/Oracle/cdm_postproc.sql b/Oracle/cdm_postproc.sql deleted file mode 100644 index 54a26d2..0000000 --- a/Oracle/cdm_postproc.sql +++ /dev/null @@ -1,37 +0,0 @@ -/* Post-processing tasks against the CDM -*/ - -/* Copy providerid from encounter table to diagnosis, procedures tables. -CDM specification says: - "Please note: This is a field replicated from the ENCOUNTER table." -*/ -merge into diagnosis d -using encounter e - on (d.encounterid = e.encounterid) -when matched then update set d.providerid = e.providerid; - -merge into procedures p -using encounter e - on (p.encounterid = e.encounterid) -when matched then update set p.providerid = e.providerid; - -merge into prescribing p -using encounter e - on (p.encounterid = e.encounterid) -when matched then update set p.rx_providerid = e.providerid; - -/* Currently in HERON, we have hight in cm and weight in oz (from visit vitals). -The CDM wants height in inches and weight in pounds. */ -update vital v set v.ht = v.ht / 2.54; -update vital v set v.wt = v.wt / 16; - -/* Remove rows from the PRESCRIBING table where RX_* fields are null - TODO: Remove this when fixed in HERON - */ -delete -from pcornet_cdm.prescribing -where rx_basis is null - and rx_quantity is null - and rx_frequency is null - and rx_refills is null -; \ No newline at end of file diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 2032aec..730cf1c 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -123,7 +123,7 @@ EOF fi -########### Run Post Proc Cleanup / Stats / Tests ########### +########### Run Stats / Tests ########### sqlplus /nolog < Date: Tue, 6 Jun 2017 09:30:38 -0500 Subject: [PATCH 270/507] Make sure that intermediary transform tables exist before transform is run --- Oracle/PCORNetInit.sql | 848 ++++++++++++++++++++++++----------------- 1 file changed, 491 insertions(+), 357 deletions(-) diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index 92d554e..899ba1a 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -1,3 +1,8 @@ + +-------------------------------------------------------------------------------- +-- HELPER FUNCTIONS AND PROCEDURES +-------------------------------------------------------------------------------- + create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS BEGIN DBMS_STATS.GATHER_TABLE_STATS ( @@ -10,6 +15,7 @@ create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS END GATHER_TABLE_STATS; / + create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS BEGIN EXECUTE IMMEDIATE sqlstring; @@ -18,6 +24,7 @@ create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS END PMN_DROPSQL; / + create or replace FUNCTION PMN_IFEXISTS(objnamestr VARCHAR2, objtypestr VARCHAR2) RETURN BOOLEAN AS cnt NUMBER; BEGIN @@ -47,6 +54,7 @@ BEGIN END PMN_ExecuateSQL; / + --ACK: http://dba.stackexchange.com/questions/9441/how-to-catch-and-handle-only-specific-oracle-exceptions create or replace procedure create_error_table(table_name varchar2) as sqltext varchar2(4000); @@ -67,8 +75,30 @@ end; / +create or replace FUNCTION GETDATAMARTID RETURN VARCHAR2 IS +BEGIN + RETURN '&&datamart_id'; +END; +/ + + +CREATE OR REPLACE FUNCTION GETDATAMARTNAME RETURN VARCHAR2 AS +BEGIN + RETURN '&&datamart_name'; +END; +/ + + +CREATE OR REPLACE FUNCTION GETDATAMARTPLATFORM RETURN VARCHAR2 AS +BEGIN + RETURN '02'; -- 01 is MSSQL, 02 is Oracle +END; +/ +-------------------------------------------------------------------------------- +-- I2B2 SYNONYMS, VIEWS, AND INTERMEDIARY TABLES +-------------------------------------------------------------------------------- CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT @@ -95,7 +125,6 @@ create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE Date: Tue, 6 Jun 2017 11:07:44 -0500 Subject: [PATCH 271/507] Replaced drop/create statements with truncate/insert, and updated indexes --- Oracle/PCORNetLoader_ora.sql | 311 ++++++++++++++++------------------- 1 file changed, 141 insertions(+), 170 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 782d0fa..8d6b9ba 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -158,7 +158,7 @@ union --6 -- NS, NR, nH begin pcornet_popcodelist; -PMN_DROPSQL('drop index demographic_patid'); +PMN_DROPSQL('drop index demographic_idx'); execute immediate 'truncate table demographic'; @@ -172,7 +172,7 @@ FETCH getsql INTO sqltext; END LOOP; CLOSE getsql; -execute immediate 'create index demographic_patid on demographic (PATID)'; +execute immediate 'create index demographic_idx on demographic (PATID)'; GATHER_TABLE_STATS('DEMOGRAPHIC'); end PCORNetDemographic; @@ -188,27 +188,26 @@ ORA-00904: "FACILITY_ID": invalid identifier ORA-00904: "LOCATION_ZIP": invalid identifier */ create or replace procedure PCORNetEncounter as - -sqltext varchar2(4000); begin -PMN_DROPSQL('drop index encounter_patid'); -PMN_DROPSQL('drop index encounter_encounterid'); +PMN_DROPSQL('drop index encounter_idx'); +PMN_DROPSQL('drop index drg_idx'); execute immediate 'truncate table encounter'; +execute immediate 'truncate table drg'; -PMN_DROPSQL('DROP TABLE drg'); -- associated indexes will be dropped as well - -sqltext := 'create table drg as '|| - 'select * from '|| - '(select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from '|| - '(select patient_num,encounter_num,drg_type,max(drg) drg from '|| - '(select distinct f.patient_num,encounter_num,SUBSTR(c_fullname,22,2) drg_type,SUBSTR(pcori_basecode,INSTR(pcori_basecode, '':'')+1,3) drg from i2b2fact f '|| - 'inner join demographic d on f.patient_num=d.patid '|| - 'inner join pcornet_enc enc on enc.c_basecode = f.concept_cd '|| - 'and enc.c_fullname like ''\PCORI\ENCOUNTER\DRG\%'') drg1 group by patient_num,encounter_num,drg_type) drg) drg '|| - 'where rn=1'; -PMN_EXECUATESQL(sqltext); +insert into drg +select * from +(select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from +(select patient_num,encounter_num,drg_type,max(drg) drg from +(select distinct f.patient_num,encounter_num,SUBSTR(c_fullname,22,2) drg_type,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,3) drg from i2b2fact f +inner join demographic d on f.patient_num=d.patid +inner join pcornet_enc enc on enc.c_basecode = f.concept_cd +and enc.c_fullname like ''\PCORI\ENCOUNTER\DRG\%'') drg1 group by patient_num,encounter_num,drg_type) drg) drg +where rn=1; + +execute immediate 'create index drg_idx on drg (patient_num, encounter_num)'; +GATHER_TABLE_STATS('drg'); insert into encounter(PATID,ENCOUNTERID,admit_date ,ADMIT_TIME , DISCHARGE_DATE ,DISCHARGE_TIME ,PROVIDERID ,FACILITY_LOCATION @@ -236,8 +235,7 @@ left outer join inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num; -execute immediate 'create index encounter_patid on encounter (PATID)'; -execute immediate 'create index encounter_encounterid on encounter (ENCOUNTERID)'; +execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; GATHER_TABLE_STATS('ENCOUNTER'); end PCORNetEncounter; @@ -245,43 +243,38 @@ end PCORNetEncounter; - create or replace procedure PCORNetDiagnosis as -sqltext varchar2(4000); begin -PMN_DROPSQL('drop index diagnosis_patid'); -PMN_DROPSQL('drop index diagnosis_encounterid'); +PMN_DROPSQL('drop index diagnosis_idx'); +PMN_DROPSQL('drop index sourcefact_idx'); +PMN_DROPSQL('drop index pdxfact_idx'); execute immediate 'truncate table diagnosis'; +execute immediate 'truncate table sourcefact'; +execute immediate 'truncate table pdxfact'; -PMN_DROPSQL('DROP TABLE sourcefact'); -- associated indexes will be dropped as well - -sqltext := 'create table sourcefact as '|| - 'select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname '|| - 'from i2b2fact factline '|| - 'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| - 'inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode '|| - 'where dxsource.c_fullname like ''\PCORI_MOD\CONDITION_OR_DX\%'''; -PMN_EXECUATESQL(sqltext); +insert into table sourcefact + select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname + from i2b2fact factline + inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + where dxsource.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\%'; execute immediate 'create index sourcefact_idx on sourcefact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('SOURCEFACT'); -PMN_DROPSQL('DROP TABLE pdxfact'); - -sqltext := 'create table pdxfact as '|| - 'select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode pdxsource,dxsource.c_fullname '|| - 'from i2b2fact factline '|| - 'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| - 'inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode '|| - 'and dxsource.c_fullname like ''\PCORI_MOD\PDX\%'''; -PMN_EXECUATESQL(sqltext); +insert into table pdxfact + select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode pdxsource,dxsource.c_fullname + from i2b2fact factline + inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + and dxsource.c_fullname like '\PCORI_MOD\PDX\%'; execute immediate 'create index pdxfact_idx on pdxfact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('PDXFACT'); -insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, pdx) +insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, pdx) /* KUMC started billing with ICD10 on Oct 1, 2015. */ with icd10_transition as ( select date '2015-10-01' as cutoff from dual @@ -347,9 +340,7 @@ where (sourcefact.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\%' or so -- order by enc.admit_date desc ; - -execute immediate 'create index diagnosis_patid on diagnosis (PATID)'; -execute immediate 'create index diagnosis_encounterid on diagnosis (ENCOUNTERID)'; +execute immediate 'create index diagnosis_idx on diagnosis (PATID, ENCOUNTERID)'; GATHER_TABLE_STATS('DIAGNOSIS'); end PCORNetDiagnosis; @@ -357,68 +348,61 @@ end PCORNetDiagnosis; - - create or replace procedure PCORNetCondition as -sqltext varchar2(4000); begin -PMN_DROPSQL('drop index condition_patid'); -PMN_DROPSQL('drop index condition_encounterid'); +PMN_DROPSQL('drop index condition_idx'); +PMN_DROPSQL('drop index sourcefact2_idx'); execute immediate 'truncate table condition'; +execute immediate 'truncate table sourcefact2'; PMN_DROPSQL('DROP TABLE sourcefact2'); -sqltext := 'create table sourcefact2 as '|| - 'select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname '|| - 'from i2b2fact factline '|| - 'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| - 'inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode '|| - 'where dxsource.c_fullname like ''\PCORI_MOD\CONDITION_OR_DX\%'''; -PMN_EXECUATESQL(sqltext); +insert into sourcefact2 + select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname + from i2b2fact factline + inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + where dxsource.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\%'; + +execute immediate 'create index sourcefact2_idx on sourcefact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('SOURCEFACT2'); create_error_table('CONDITION'); -sqltext := 'insert into condition (patid, encounterid, report_date, resolve_date, condition, condition_type, condition_status, condition_source) '|| -'select distinct factline.patient_num, min(factline.encounter_num) encounterid, min(factline.start_date) report_date, NVL(max(factline.end_date),null) resolve_date, diag.pcori_basecode, '|| -'SUBSTR(diag.c_fullname,18,2) condition_type, '|| -' NVL2(max(factline.end_date) , ''RS'', ''NI'') condition_status, '|| -- Imputed so might not be entirely accurate -' NVL(SUBSTR(max(dxsource),INSTR(max(dxsource), '':'')+1,2),''NI'') condition_source '|| -'from i2b2fact factline '|| -'inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| -'inner join pcornet_diag diag on diag.c_basecode = factline.concept_cd '|| -' left outer join sourcefact2 sf '|| -'on factline.patient_num=sf.patient_num '|| -'and factline.encounter_num=sf.encounter_num '|| -'and factline.provider_id=sf.provider_id '|| -'and factline.concept_cd=sf.concept_Cd '|| -'and factline.start_date=sf.start_Date '|| -'where diag.c_fullname like ''\PCORI\DIAGNOSIS\%'' '|| -'and sf.c_fullname like ''\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\%'' '|| -'group by factline.patient_num, diag.pcori_basecode, diag.c_fullname ' || -'log errors into ERR$_CONDITION reject limit unlimited' +insert into condition (patid, encounterid, report_date, resolve_date, condition, condition_type, condition_status, condition_source) +select distinct factline.patient_num, min(factline.encounter_num) encounterid, min(factline.start_date) report_date, NVL(max(factline.end_date),null) resolve_date, diag.pcori_basecode, +SUBSTR(diag.c_fullname,18,2) condition_type, + NVL2(max(factline.end_date) , 'RS', 'NI') condition_status, -- Imputed so might not be entirely accurate + NVL(SUBSTR(max(dxsource),INSTR(max(dxsource), ':')+1,2),'NI') condition_source +from i2b2fact factline +inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num +inner join pcornet_diag diag on diag.c_basecode = factline.concept_cd + left outer join sourcefact2 sf +on factline.patient_num=sf.patient_num +and factline.encounter_num=sf.encounter_num +and factline.provider_id=sf.provider_id +and factline.concept_cd=sf.concept_Cd +and factline.start_date=sf.start_Date +where diag.c_fullname like '\PCORI\DIAGNOSIS\%' +and sf.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\%' +group by factline.patient_num, diag.pcori_basecode, diag.c_fullname +log errors into ERR$_CONDITION reject limit unlimited ; -PMN_EXECUATESQL(sqltext); - -execute immediate 'create index condition_patid on condition (PATID)'; -execute immediate 'create index condition_encounterid on condition (ENCOUNTERID)'; +execute immediate 'create index condition_idx on condition (PATID, ENCOUNTERID)'; +GATHER_TABLE_STATS('CONDITION'); end PCORNetCondition; / - - - - create or replace procedure PCORNetProcedure as begin -PMN_DROPSQL('drop index procedures_patid'); -PMN_DROPSQL('drop index procedures_encounterid'); +PMN_DROPSQL('drop index procedures_idx'); execute immediate 'truncate table procedures'; @@ -433,8 +417,7 @@ from i2b2fact fact inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd where pr.c_fullname like '\PCORI\PROCEDURE\%'; -execute immediate 'create index procedures_patid on procedures (PATID)'; -execute immediate 'create index procedures_encounterid on procedures (ENCOUNTERID)'; +execute immediate 'create index procedures_patid on procedures (PATID, ENCOUNTERID)'; GATHER_TABLE_STATS('PROCEDURES'); end PCORNetProcedure; @@ -449,8 +432,7 @@ end PCORNetProcedure; create or replace procedure PCORNetVital as begin -PMN_DROPSQL('drop index vital_patid'); -PMN_DROPSQL('drop index vital_encounterid'); +PMN_DROPSQL('drop index vital_idx'); execute immediate 'truncate table vital'; @@ -527,8 +509,7 @@ where ht is not null or tobacco is not null group by patid, encounterid, measure_date, measure_time, admit_date) y; -execute immediate 'create index vital_patid on vital (PATID)'; -execute immediate 'create index vital_encounterid on vital (ENCOUNTERID)'; +execute immediate 'create index vital_idx on vital (PATID)'; GATHER_TABLE_STATS('VITAL'); end PCORNetVital; @@ -542,7 +523,7 @@ end PCORNetVital; create or replace procedure PCORNetEnroll as begin -PMN_DROPSQL('drop index enrollment_patid'); +PMN_DROPSQL('drop index enrollment_idx'); execute immediate 'truncate table enrollment'; @@ -566,7 +547,7 @@ from enrolled enr join i2b2visit visit on enr.patient_num = visit.patient_num group by visit.patient_num; -execute immediate 'create index enrollment_patid on enrollment (PATID)'; +execute immediate 'create index enrollment_idx on enrollment (PATID)'; GATHER_TABLE_STATS('ENROLLMENT'); end PCORNetEnroll; @@ -574,37 +555,32 @@ end PCORNetEnroll; create or replace procedure PCORNetLabResultCM as -sqltext varchar2(4000); begin -PMN_DROPSQL('drop index lab_result_cm_patid'); -PMN_DROPSQL('drop index lab_result_cm_encounterid'); - -execute immediate 'truncate table lab_result_cm'; +PMN_DROPSQL('drop index lab_result_cm_idx'); +PMN_DROPSQL('drop index priority_idx'); +PMN_DROPSQL('drop index location_idx'); -PMN_DROPSQL('DROP TABLE priority'); +execute immediate 'truncate table priority'; +execute immediate 'truncate table location'; +execute immediate 'truncate table lab_result_cm'; -sqltext := 'create table priority as '|| -'(select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode PRIORITY '|| -'from i2b2fact '|| -'inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num '|| -'inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode '|| -'where c_fullname LIKE ''\PCORI_MOD\PRIORITY\%'') '; - -PMN_EXECUATESQL(sqltext); +insert into priority +select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode PRIORITY +from i2b2fact +inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num +inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode +where c_fullname LIKE '\PCORI_MOD\PRIORITY\%'); execute immediate 'create index priority_idx on priority (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('PRIORITY'); -PMN_DROPSQL('DROP TABLE location'); -sqltext := 'create table location as '|| -'(select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode RESULT_LOC '|| -'from i2b2fact '|| -'inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num '|| -'inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode '|| -'where c_fullname LIKE ''\PCORI_MOD\RESULT_LOC\%'') '; - -PMN_EXECUATESQL(sqltext); +insert into location +select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode RESULT_LOC +from i2b2fact +inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num +inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode +where c_fullname LIKE '\PCORI_MOD\RESULT_LOC\%'; execute immediate 'create index location_idx on location (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('LOCATION'); @@ -706,8 +682,8 @@ and M.start_date=l.start_Date WHERE m.ValType_Cd in ('N','T') and m.MODIFIER_CD='@'; -execute immediate 'create index lab_result_cm_patid on lab_result_cm (PATID)'; -execute immediate 'create index lab_result_cm_encounterid on lab_result_cm (ENCOUNTERID)'; +execute immediate 'create index lab_result_cm_idx on lab_result_cm (PATID, ENCOUNTERID)'; +GATHER_TABLE_STATS('LAB_RESULT_CM'); END PCORNetLabResultCM; / @@ -745,71 +721,68 @@ end PCORNetHarvest; create or replace procedure PCORNetPrescribing as -sqltext varchar2(4000); begin -PMN_DROPSQL('drop index prescribing_patid'); -PMN_DROPSQL('drop index prescribing_encounterid'); +PMN_DROPSQL('drop index prescribing_idx'); +PMN_DROPSQL('drop index basis_idx'); +PMN_DROPSQL('drop index freq_idx'); +PMN_DROPSQL('drop index quantity_idx'); +PMN_DROPSQL('drop index refills_idx'); +PMN_DROPSQL('drop index supply_idx'); execute immediate 'truncate table prescribing'; - -PMN_DROPSQL('DROP TABLE basis'); -sqltext := 'create table basis as '|| -'(select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact basis '|| -' inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num '|| -' join pcornet_med basiscode '|| -' on basis.modifier_cd = basiscode.c_basecode '|| -' and basiscode.c_fullname like ''\PCORI_MOD\RX_BASIS\%'') '; -PMN_EXECUATESQL(sqltext); +execute immediate 'truncate table basis'; +execute immediate 'truncate table freq'; +execute immediate 'truncate table quantity'; +execute immediate 'truncate table refills'; +execute immediate 'truncate table supply'; + +insert into table basis +select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact basis + inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num + join pcornet_med basiscode + on basis.modifier_cd = basiscode.c_basecode + and basiscode.c_fullname like '\PCORI_MOD\RX_BASIS\%'; execute immediate 'create unique index basis_idx on basis (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('BASIS'); -PMN_DROPSQL('DROP TABLE freq'); -sqltext := 'create table freq as '|| -'(select pcori_basecode,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact freq '|| -' inner join encounter enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num '|| -' join pcornet_med freqcode '|| -' on freq.modifier_cd = freqcode.c_basecode '|| -' and freqcode.c_fullname like ''\PCORI_MOD\RX_FREQUENCY\%'') '; -PMN_EXECUATESQL(sqltext); +insert into table freq +select pcori_basecode,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact freq + inner join encounter enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num + join pcornet_med freqcode + on freq.modifier_cd = freqcode.c_basecode + and freqcode.c_fullname like '\PCORI_MOD\RX_FREQUENCY\%'; execute immediate 'create unique index freq_idx on freq (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('FREQ'); -PMN_DROPSQL('DROP TABLE quantity'); -sqltext := 'create table quantity as '|| -'(select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact quantity '|| -' inner join encounter enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num '|| -' join pcornet_med quantitycode '|| -' on quantity.modifier_cd = quantitycode.c_basecode '|| -' and quantitycode.c_fullname like ''\PCORI_MOD\RX_QUANTITY\'') '; - -PMN_EXECUATESQL(sqltext); +insert into table quantity +select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact quantity + inner join encounter enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num + join pcornet_med quantitycode + on quantity.modifier_cd = quantitycode.c_basecode + and quantitycode.c_fullname like '\PCORI_MOD\RX_QUANTITY\'; execute immediate 'create unique index quantity_idx on quantity (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('QUANTITY'); -PMN_DROPSQL('DROP TABLE refills'); -sqltext := 'create table refills as '|| -'(select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact refills '|| -' inner join encounter enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num '|| -' join pcornet_med refillscode '|| -' on refills.modifier_cd = refillscode.c_basecode '|| -' and refillscode.c_fullname like ''\PCORI_MOD\RX_REFILLS\'') '; -PMN_EXECUATESQL(sqltext); +insert into table refills +select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact refills + inner join encounter enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num + join pcornet_med refillscode + on refills.modifier_cd = refillscode.c_basecode + and refillscode.c_fullname like '\PCORI_MOD\RX_REFILLS\'; execute immediate 'create unique index refills_idx on refills (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('REFILLS'); -PMN_DROPSQL('DROP TABLE supply'); -sqltext := 'create table supply as '|| -'(select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact supply '|| -' inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| -' join pcornet_med supplycode '|| -' on supply.modifier_cd = supplycode.c_basecode '|| -' and supplycode.c_fullname like ''\PCORI_MOD\RX_DAYS_SUPPLY\'') '; -PMN_EXECUATESQL(sqltext); +insert into table supply +select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact supply + inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num + join pcornet_med supplycode + on supply.modifier_cd = supplycode.c_basecode + and supplycode.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\'; execute immediate 'create unique index supply_idx on supply (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('SUPPLY'); @@ -878,8 +851,7 @@ inner join encounter enc on enc.encounterid = m.encounter_Num where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); -execute immediate 'create index prescribing_patid on prescribing (PATID)'; -execute immediate 'create index prescribing_encounterid on prescribing (ENCOUNTERID)'; +execute immediate 'create index prescribing_idx on prescribing (PATID, ENCOUNTERID)'; GATHER_TABLE_STATS('PRESCRIBING'); end PCORNetPrescribing; @@ -889,10 +861,9 @@ end PCORNetPrescribing; create or replace procedure PCORNetDispensing as -sqltext varchar2(4000); begin -PMN_DROPSQL('drop index dispensing_patid'); +PMN_DROPSQL('drop index dispensing_idx'); execute immediate 'truncate table dispensing'; /* @@ -1004,7 +975,8 @@ inner join encounter enc on enc.encounterid = m.encounter_Num group by m.encounter_num ,m.patient_num, m.start_date, mo.pcori_ndc; */ -execute immediate 'create index dispensing_patid on dispensing (PATID)'; +execute immediate 'create index dispensing_idx on dispensing (PATID)'; +GATHER_TABLE_STATS('DISPENSING'); end PCORNetDispensing; / @@ -1020,7 +992,6 @@ end PCORNetDispensing; create or replace procedure PCORNetDeath as - begin execute immediate 'truncate table death'; From ccc4b1c999de8c3a4c3b8e684937312b2c658e53 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 6 Jun 2017 11:12:46 -0500 Subject: [PATCH 272/507] Removing ontology mapping from this repo and moving to h2p-mapping repo --- Oracle/PCORI_MEDS_SCHEMA_CHANGE_ora.sql | 50 -- Oracle/gather_table_stats.sql | 20 - Oracle/lab_loinc_mapping.csv | 156 ------ Oracle/load_pcornet_mapping.sh | 29 - Oracle/ontology_fix_script_ora.sql | 41 -- Oracle/pcornet_mapping.csv | 241 --------- Oracle/pcornet_mapping.ctl | 8 - Oracle/pcornet_mapping.sql | 692 ------------------------ Oracle/pcornet_mapping_ddl.sql | 4 - Oracle/pcornet_ontology_updates.csv | 9 - Oracle/pcornet_ontology_updates.ctl | 32 -- Oracle/pcornet_ontology_updates_ddl.sql | 29 - 12 files changed, 1311 deletions(-) delete mode 100644 Oracle/PCORI_MEDS_SCHEMA_CHANGE_ora.sql delete mode 100644 Oracle/gather_table_stats.sql delete mode 100644 Oracle/lab_loinc_mapping.csv delete mode 100644 Oracle/load_pcornet_mapping.sh delete mode 100644 Oracle/ontology_fix_script_ora.sql delete mode 100644 Oracle/pcornet_mapping.csv delete mode 100644 Oracle/pcornet_mapping.ctl delete mode 100644 Oracle/pcornet_mapping.sql delete mode 100644 Oracle/pcornet_mapping_ddl.sql delete mode 100644 Oracle/pcornet_ontology_updates.csv delete mode 100644 Oracle/pcornet_ontology_updates.ctl delete mode 100644 Oracle/pcornet_ontology_updates_ddl.sql diff --git a/Oracle/PCORI_MEDS_SCHEMA_CHANGE_ora.sql b/Oracle/PCORI_MEDS_SCHEMA_CHANGE_ora.sql deleted file mode 100644 index 139f479..0000000 --- a/Oracle/PCORI_MEDS_SCHEMA_CHANGE_ora.sql +++ /dev/null @@ -1,50 +0,0 @@ -alter table pcornet_med add PCORI_NDC varchar2(12); -UPDATE PCORNET_MED set pcori_ndc = pcori_basecode where length(pcori_basecode)=11 and c_hlevel>2 and LOWER(sourcesystem_cd) not in ('integration_tool') -and pcori_basecode not like 'N%'; - -alter table pcornet_med add PCORI_CUI varchar2(8); -UPDATE pcornet_med set PCORI_CUI = PCORI_BASECODE where length(pcori_basecode)<11 and c_hlevel>2 and LOWER(sourcesystem_cd) not in ('integration_tool') -and pcori_basecode not like 'N%' and m_applied_path='@'; - - -create table CUI_T as -select * from -( -with -cui(c_fullname,pcori_cui,c_hlevel) as -( - select c_fullname,pcori_cui,c_hlevel from pcornet_med where pcori_cui is not null - union all - select m.c_fullname, cui.pcori_cui, m.c_hlevel - from pcornet_med m inner join cui on cui.c_fullname=m.c_path where m.pcori_cui is null -), -cuid as -( select c_fullname, pcori_cui, row_number() over (partition by C_FULLNAME order by c_hlevel desc)as rowno from cui) - -select DISTINCT cuid.c_fullname as c_fullname, cuid.pcori_cui as pcori_cui from cuid where cuid.rowno=1 -); - -update pcornet_med x set x.pcori_cui = - (select pcori_cui from CUI_T t where x.c_fullname = t.c_fullname) -where x.pcori_cui is null; - - - -create table NDC_T as -select * from -( -with ndc(c_fullname,pcori_ndc,c_hlevel) as -( - select c_fullname,pcori_ndc,c_hlevel from pcornet_med where pcori_ndc is not null - union all - select m.c_fullname,ndc.pcori_ndc,m.c_hlevel from pcornet_med m - inner join ndc on ndc.c_fullname=m.c_path where m.pcori_ndc is null -), -ndcd as ( select c_fullname,pcori_ndc, row_number() over (partition by C_FULLNAME order by c_hlevel desc) row from ndc) - -select DISTINCT ndcd.c_fullname as c_fullname, ndcd.pcori_ndc as pcori_ndc from ndcd where ndcd.row=1 -); - -update pcornet_med x set x.pcori_ndc = - (select pcori_ndc from NDC_T t where x.c_fullname = t.c_fullname) -where x.pcori_ndc is null; \ No newline at end of file diff --git a/Oracle/gather_table_stats.sql b/Oracle/gather_table_stats.sql deleted file mode 100644 index 94f1a5e..0000000 --- a/Oracle/gather_table_stats.sql +++ /dev/null @@ -1,20 +0,0 @@ -/* Metadata table statistics - experimentation shows vastly better performance -if this is done before the CDM transform. - -ACK: http://stackoverflow.com/questions/2242024/for-each-string-execute-a-function-procedure -*/ -declare -table_list sys.dbms_debug_vc2coll := sys.dbms_debug_vc2coll( - 'PCORNET_DEMO', 'PCORNET_DIAG', 'PCORNET_ENC', 'PCORNET_ENROLL', 'PCORNET_LAB', - 'PCORNET_MED', 'PCORNET_PROC', 'PCORNET_VITAL'); -begin - for t in table_list.first .. table_list.last - loop - DBMS_STATS.GATHER_TABLE_STATS ( - ownname => '"&&i2b2_meta_schema"', - tabname => table_list(t), - estimate_percent => 40 - ); - end loop; -end; -/ diff --git a/Oracle/lab_loinc_mapping.csv b/Oracle/lab_loinc_mapping.csv deleted file mode 100644 index 9338321..0000000 --- a/Oracle/lab_loinc_mapping.csv +++ /dev/null @@ -1,156 +0,0 @@ -"LOINC_CODE","LAB_NAME","PCORI_SPECIMEN_SOURCE" -"13457-7","LDL","" -"18261-8","LDL","" -"2089-1","LDL","" -"47213-4","LDL","SERUM, PLASMA, or SR_PLS" -"47213-4","LDL","SERUM, PLASMA, or SR_PLS" -"47213-4","LDL","SERUM, PLASMA, or SR_PLS" -"49132-4","LDL","" -"17855-8","A1C","" -"17856-6","A1C","BLOOD" -"4548-4","A1C","BLOOD" -"59261-8","A1C","" -"62388-4","A1C","" -"2157-6","CK","SERUM, PLASMA, or SR_PLS" -"2157-6","CK","SERUM, PLASMA, or SR_PLS" -"2157-6","CK","SERUM, PLASMA, or SR_PLS" -"2157-6","CK","SERUM, PLASMA, or SR_PLS" -"50756-6","CK","BLOOD" -"50756-6","CK","BLOOD" -"50756-6","CK","BLOOD" -"50756-6","CK","BLOOD" -"13969-1","CK_MB","SERUM, PLASMA, or SR_PLS" -"13969-1","CK_MB","SERUM, PLASMA, or SR_PLS" -"13969-1","CK_MB","SERUM, PLASMA, or SR_PLS" -"13969-1","CK_MB","SERUM, PLASMA, or SR_PLS" -"2154-3","CK_MB","SERUM, PLASMA, or SR_PLS" -"2154-3","CK_MB","SERUM, PLASMA, or SR_PLS" -"2154-3","CK_MB","SERUM, PLASMA, or SR_PLS" -"2154-3","CK_MB","SERUM, PLASMA, or SR_PLS" -"32673-6","CK_MB","SERUM, PLASMA, or SR_PLS" -"32673-6","CK_MB","SERUM, PLASMA, or SR_PLS" -"32673-6","CK_MB","SERUM, PLASMA, or SR_PLS" -"32673-6","CK_MB","SERUM, PLASMA, or SR_PLS" -"49551-5","CK_MB","BLOOD" -"49551-5","CK_MB","BLOOD" -"49551-5","CK_MB","BLOOD" -"49551-5","CK_MB","BLOOD" -"12187-1","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12187-1","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12187-1","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12187-1","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12189-7","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12189-7","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12189-7","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12189-7","CK_MBI","SERUM, PLASMA, or SR_PLS" -"20569-0","CK_MBI","SERUM, PLASMA, or SR_PLS" -"20569-0","CK_MBI","SERUM, PLASMA, or SR_PLS" -"20569-0","CK_MBI","SERUM, PLASMA, or SR_PLS" -"20569-0","CK_MBI","SERUM, PLASMA, or SR_PLS" -"49136-5","CK_MBI","SERUM, PLASMA, or SR_PLS" -"49136-5","CK_MBI","SERUM, PLASMA, or SR_PLS" -"49136-5","CK_MBI","SERUM, PLASMA, or SR_PLS" -"49136-5","CK_MBI","SERUM, PLASMA, or SR_PLS" -"2160-0","CREATININE","SERUM, PLASMA, or SR_PLS" -"38483-4","CREATININE","BLOOD" -"14682-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"14682-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"14682-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"14682-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"21232-4","CREATININE","BLOOD" -"21232-4","CREATININE","BLOOD" -"21232-4","CREATININE","BLOOD" -"21232-4","CREATININE","BLOOD" -"35203-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"35203-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"35203-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"35203-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"44784-7","CREATININE","SERUM, PLASMA, or SR_PLS" -"44784-7","CREATININE","SERUM, PLASMA, or SR_PLS" -"44784-7","CREATININE","SERUM, PLASMA, or SR_PLS" -"44784-7","CREATININE","SERUM, PLASMA, or SR_PLS" -"59826-8","CREATININE","BLOOD" -"59826-8","CREATININE","BLOOD" -"59826-8","CREATININE","BLOOD" -"59826-8","CREATININE","BLOOD" -"14775-1","HGB","BLOOD" -"14775-1","HGB","BLOOD" -"14775-1","HGB","BLOOD" -"14775-1","HGB","BLOOD" -"20509-6","HGB","BLOOD" -"20509-6","HGB","BLOOD" -"20509-6","HGB","BLOOD" -"20509-6","HGB","BLOOD" -"30313-1","HGB","BLOOD" -"30313-1","HGB","BLOOD" -"30313-1","HGB","BLOOD" -"30313-1","HGB","BLOOD" -"30350-3","HGB","BLOOD" -"30350-3","HGB","BLOOD" -"30350-3","HGB","BLOOD" -"30350-3","HGB","BLOOD" -"30351-1","HGB","BLOOD" -"30351-1","HGB","BLOOD" -"30351-1","HGB","BLOOD" -"30351-1","HGB","BLOOD" -"30352-9","HGB","BLOOD" -"30352-9","HGB","BLOOD" -"30352-9","HGB","BLOOD" -"30352-9","HGB","BLOOD" -"55782-7","HGB","BLOOD" -"55782-7","HGB","BLOOD" -"55782-7","HGB","BLOOD" -"55782-7","HGB","BLOOD" -"59260-0","HGB","BLOOD" -"59260-0","HGB","BLOOD" -"59260-0","HGB","BLOOD" -"59260-0","HGB","BLOOD" -"718-7","HGB","BLOOD" -"34714-6","INR","BLOOD" -"34714-6","INR","BLOOD" -"34714-6","INR","BLOOD" -"34714-6","INR","BLOOD" -"46418-0","INR","BLOOD" -"46418-0","INR","BLOOD" -"46418-0","INR","BLOOD" -"46418-0","INR","BLOOD" -"6301-6","INR","PPP" -"6301-6","INR","PPP" -"6301-6","INR","PPP" -"6301-6","INR","PPP" -"10839-9","TROP_I","SERUM, PLASMA, or SR_PLS" -"10839-9","TROP_I","SERUM, PLASMA, or SR_PLS" -"10839-9","TROP_I","SERUM, PLASMA, or SR_PLS" -"10839-9","TROP_I","SERUM, PLASMA, or SR_PLS" -"16255-2","TROP_I","SERUM, PLASMA, or SR_PLS" -"16255-2","TROP_I","SERUM, PLASMA, or SR_PLS" -"16255-2","TROP_I","SERUM, PLASMA, or SR_PLS" -"16255-2","TROP_I","SERUM, PLASMA, or SR_PLS" -"42757-5","TROP_I","BLOOD" -"42757-5","TROP_I","BLOOD" -"42757-5","TROP_I","BLOOD" -"42757-5","TROP_I","BLOOD" -"49563-0","TROP_I","SERUM, PLASMA, or SR_PLS" -"49563-0","TROP_I","SERUM, PLASMA, or SR_PLS" -"49563-0","TROP_I","SERUM, PLASMA, or SR_PLS" -"49563-0","TROP_I","SERUM, PLASMA, or SR_PLS" -"33204-9","TROP_T_QL","SERUM, PLASMA, or SR_PLS" -"33204-9","TROP_T_QL","SERUM, PLASMA, or SR_PLS" -"33204-9","TROP_T_QL","SERUM, PLASMA, or SR_PLS" -"33204-9","TROP_T_QL","SERUM, PLASMA, or SR_PLS" -"48426-1","TROP_T_QL","BLOOD" -"48426-1","TROP_T_QL","BLOOD" -"48426-1","TROP_T_QL","BLOOD" -"48426-1","TROP_T_QL","BLOOD" -"48425-3","TROP_T_QN","BLOOD" -"48425-3","TROP_T_QN","BLOOD" -"48425-3","TROP_T_QN","BLOOD" -"48425-3","TROP_T_QN","BLOOD" -"6597-9","TROP_T_QN","BLOOD" -"6597-9","TROP_T_QN","BLOOD" -"6597-9","TROP_T_QN","BLOOD" -"6597-9","TROP_T_QN","BLOOD" -"6598-7","TROP_T_QN","SERUM, PLASMA, or SR_PLS" -"6598-7","TROP_T_QN","SERUM, PLASMA, or SR_PLS" -"6598-7","TROP_T_QN","SERUM, PLASMA, or SR_PLS" -"6598-7","TROP_T_QN","SERUM, PLASMA, or SR_PLS" diff --git a/Oracle/load_pcornet_mapping.sh b/Oracle/load_pcornet_mapping.sh deleted file mode 100644 index 651d6cc..0000000 --- a/Oracle/load_pcornet_mapping.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -set -e - -# Expected environment variables -# Database SID -# export sid= - -# User and password for CDM user -# export pcornet_cdm_user= -# export pcornet_cdm= - -# Create/Load the local path mapping and ontology update tables -sqlplus /nolog < 1; - -How many of each spec_order? - -select count(*), spec_order, dose_pref from medication_id_to_best_rxcui -group by spec_order, dose_pref order by 1 desc; - -27825 1) Semantic generic clinical drug (SCD) 1) Clarity -17527 1) Semantic generic clinical drug (SCD) 2) GCN - 3475 *null* lots of IVP. TODO: med mixes - 2888 9) HERON mapping misc. 2) GCN - 2550 9) HERON mapping misc. 1) Clarity - 421 1) Semantic generic clinical drug (SCD) 3) NDC - 295 3) Generic drug pack (GPCK) 1) Clarity - 177 3) Generic drug pack (GPCK) 2) GCN - 97 2) Semantic Branded clinical drug (SBC) 3) NDC - 4 9) HERON mapping misc. 3) NDC - 2 4) Branded drug pack (BPCK) 3) NDC - 1 3) Generic drug pack (GPCK) 3) NDC - -Be sure RXCUI for 0.4 ML ENOXAPARIN 100 MG/ML SC SYRG includes the 0.4 ML dose info: -select concept_cd, name_char, rxcui, rxnorm_str, spec_order, dose_pref from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:85052'; -- -> 854235, 1) SCD - -select concept_cd, name_char, rxcui, rxnorm_str, spec_order, dose_pref from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:85051'; -- -> 854248, 1) SCD - -This one goes from "LEVOTHYROXINE PO" to "Levothyroxine Sodium 0.1 MG Oral Tablet"; is that right? -select concept_cd, name_char, rxcui, rxnorm_str, spec_order, dose_pref from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:150171'; -- -> 892246, 1) SCD -*/ - - -insert into "&&i2b2_meta_schema".PCORNET_MED -with -terms_rx as ( - select - best.rxcui mapped_rxcui, ht.* - from - "&&i2b2_meta_schema"."&&terms_table" ht - left join medication_id_to_best_rxcui best on best.concept_cd = ht.c_basecode - where c_fullname like '\i2b2\Medications%' - and c_basecode not like 'NDC:%' -- We'll handle NDCs seperately below - and ht.c_visualattributes not like '%H%' - ) -select - rx.c_hlevel + 1 c_hlevel, - replace(rx.c_fullname, '\i2b2\Medications\', '\PCORI\MEDICATION\RXNORM_CUI\') c_fullname, - rx.c_name, rx.c_synonym_cd, rx.c_visualattributes, - rx.c_totalnum, rx.c_basecode, rx.c_metadataxml, rx.c_facttablecolumn, rx.c_tablename, - rx.c_columnname, rx.c_columndatatype, rx.c_operator, rx.c_dimcode, rx.c_comment, - rx.c_tooltip, rx.m_applied_path, rx.update_date, rx.download_date, rx.import_date, - rx.sourcesystem_cd, rx.valuetype_cd, rx.m_exclusion_cd, rx.c_path, rx.c_symbol, - rx.rxcui pcori_basecode, rx.rxcui pcori_cui, - -- Don't worry about NDCs for now - used for dispensing - null pcori_ndc from ( - select - trx.*, - case - when trx.mapped_rxcui is not null then trx.mapped_rxcui - when trx.c_basecode like 'RXCUI:%' then replace(trx.c_basecode, 'RXCUI:', '') - else null - end rxcui - from terms_rx trx - --order by trx.c_hlevel - ) rx; - --- Handle mapping NDC codes for DISPENSING -insert into "&&i2b2_meta_schema".PCORNET_MED -select - c_hlevel + 1 c_hlevel, - replace(c_fullname, '\i2b2\Medications\', '\PCORI\MEDICATION\RXNORM_CUI\') c_fullname, - c_name, c_synonym_cd, c_visualattributes, - c_totalnum, c_basecode, c_metadataxml, c_facttablecolumn, c_tablename, - c_columnname, c_columndatatype, c_operator, c_dimcode, c_comment, - c_tooltip, m_applied_path, update_date, download_date, import_date, - sourcesystem_cd, valuetype_cd, m_exclusion_cd, c_path, c_symbol, - null pcori_basecode, null pcori_cui, - substr(c_basecode, 5) pcori_ndc -from "&&i2b2_meta_schema"."&&terms_table" -where c_fullname like '\i2b2\Medications%' - and c_basecode like 'NDC:%' - and length(substr(c_basecode, 5)) < 12 -- Make sure we have 11 digit NDC -; - -delete -from "&&i2b2_meta_schema".PCORNET_MED where sourcesystem_cd='MAPPING'; - - -insert into "&&i2b2_meta_schema".PCORNET_MED -with med_mod_mapping as ( - select distinct pcori_path, i2b2.c_fullname, i2b2.c_basecode, i2b2.c_name - FROM "&&i2b2_meta_schema".pcornet_med - join pcornet_mapping on pcornet_mapping.PCORI_PATH = pcornet_med.c_fullname and pcornet_mapping.local_path is not null - join "&&i2b2_meta_schema"."&&terms_table" i2b2 on i2b2.c_fullname like pcornet_mapping.local_path || '%' - where i2b2.c_basecode is not null - ) -SELECT pcornet_med.C_HLEVEL+1, - pcornet_med.C_FULLNAME || med_mod_mapping.c_basecode || '\' as C_FULLNAME, - med_mod_mapping.c_basecode || ' ' || med_mod_mapping.c_name as C_NAME, - pcornet_med.C_SYNONYM_CD, - pcornet_med.C_VISUALATTRIBUTES, - pcornet_med.C_TOTALNUM, - med_mod_mapping.c_basecode as C_BASECODE, - pcornet_med.C_METADATAXML, - pcornet_med.C_FACTTABLECOLUMN, - pcornet_med.C_TABLENAME, - pcornet_med.C_COLUMNNAME, - pcornet_med.C_COLUMNDATATYPE, - pcornet_med.C_OPERATOR, - pcornet_med.C_FULLNAME || med_mod_mapping.c_basecode || '\' as C_DIMCODE, - pcornet_med.C_COMMENT, - pcornet_med.C_TOOLTIP, - pcornet_med.M_APPLIED_PATH, - pcornet_med.UPDATE_DATE, - pcornet_med.DOWNLOAD_DATE, - pcornet_med.IMPORT_DATE, - 'MAPPING' as SOURCESYSTEM_CD, - pcornet_med.VALUETYPE_CD, - pcornet_med.M_EXCLUSION_CD, - pcornet_med.C_PATH, - pcornet_med.C_SYMBOL, - pcornet_med.PCORI_BASECODE, - null pcori_cui, - null pcori_ndc -from med_mod_mapping -join "&&i2b2_meta_schema".pcornet_med on med_mod_mapping.PCORI_PATH = pcornet_med.c_fullname -; - -commit; - -/* Replace SCILHS PCORNet Labs hierarchy with our local LOINC hierarchy with - adjustment to make it "SCILHS like". -*/ - -/* Remove existing SCIHLS Labs hierarchy */ -truncate table "&&i2b2_meta_schema".pcornet_lab; - -/* Insert LOINC codes from local hierarchy, setting c_basecode to appropriate - LAB_NAME for common PCORNet labs. */ -insert into "&&i2b2_meta_schema".pcornet_lab -select distinct - lc.C_HLEVEL, - replace(lc.C_FULLNAME, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_fullname, - lc.C_NAME, - lc.C_SYNONYM_CD, - lc.C_VISUALATTRIBUTES, - lc.C_TOTALNUM, - case - when llm.lab_name is not null then 'LAB_NAME:'|| llm.lab_name - else lc.c_basecode - end c_basecode, - to_char(lc.C_METADATAXML), - lc.C_FACTTABLECOLUMN, - lc.C_TABLENAME, - lc.C_COLUMNNAME, - lc.C_COLUMNDATATYPE, - lc.C_OPERATOR, - lc.C_DIMCODE, - to_char(lc.C_COMMENT), - lc.C_TOOLTIP, - lc.M_APPLIED_PATH, - lc.UPDATE_DATE, - lc.DOWNLOAD_DATE, - lc.IMPORT_DATE, - lc.SOURCESYSTEM_CD, - lc.VALUETYPE_CD, - lc.M_EXCLUSION_CD, - lc.C_PATH, - lc.C_SYMBOL, - llm.pcori_specimen_source, - replace(lc.c_basecode, 'LOINC:', '') pcori_basecode -from "&&i2b2_meta_schema"."&&terms_table" lc -left outer join lab_loinc_mapping llm - on lc.c_basecode = ('LOINC:' || llm.loinc_code) -where lc.c_basecode like 'LOINC:%' -; - -/* Insert child KUH|COMPONENT_ID nodes, setting pcori_basecode, and c_path to - that of it's parent. */ -insert into "&&i2b2_meta_schema".pcornet_lab -with parent_loinc_codes as ( -- LOINC codes with children LOINC codes - select p_loinc.* - from "&&i2b2_meta_schema"."&&terms_table" p_loinc - join "&&i2b2_meta_schema"."&&terms_table" c_loinc - on c_loinc.c_fullname like (p_loinc.c_fullname || '%') - and p_loinc.c_basecode like 'LOINC:%' - and c_loinc.c_basecode like 'LOINC:%' - and p_loinc.c_basecode!=c_loinc.c_basecode -) -, children_loinc_codes as ( -- LOINC codes without children LOINC codes - select clc.* - from "&&i2b2_meta_schema"."&&terms_table" clc - where clc.c_basecode like 'LOINC:%' - and clc.c_basecode not in ( - select c_basecode from parent_loinc_codes - ) -) -select distinct - ccc.C_HLEVEL, - replace(ccc.C_FULLNAME, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_fullname, - ccc.C_NAME, - ccc.C_SYNONYM_CD, - ccc.C_VISUALATTRIBUTES, - ccc.C_TOTALNUM, - ccc.C_BASECODE, - to_char(ccc.C_METADATAXML), - ccc.C_FACTTABLECOLUMN, - ccc.C_TABLENAME, - ccc.C_COLUMNNAME, - ccc.C_COLUMNDATATYPE, - ccc.C_OPERATOR, - ccc.C_DIMCODE, - to_char(ccc.C_COMMENT), - ccc.C_TOOLTIP, - ccc.M_APPLIED_PATH, - ccc.UPDATE_DATE, - ccc.DOWNLOAD_DATE, - ccc.IMPORT_DATE, - ccc.SOURCESYSTEM_CD, - ccc.VALUETYPE_CD, - ccc.M_EXCLUSION_CD, - replace(clc.C_FULLNAME, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_path, - ccc.C_SYMBOL, - llm.pcori_specimen_source, - replace(clc.c_basecode, 'LOINC:', '') pcori_basecode -from children_loinc_codes clc -join "&&i2b2_meta_schema"."&&terms_table" ccc - on ccc.c_fullname like (clc.c_fullname || '%') - and ccc.c_basecode like 'KUH|COMPONENT_ID:%' -- TODO: Generalize for other sites. -left outer join lab_loinc_mapping llm - on clc.c_basecode = ('LOINC:' || llm.loinc_code) -; - -commit; - diff --git a/Oracle/pcornet_mapping_ddl.sql b/Oracle/pcornet_mapping_ddl.sql deleted file mode 100644 index 12d6e87..0000000 --- a/Oracle/pcornet_mapping_ddl.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE TABLE pcornet_mapping ( - "PCORI_PATH" VARCHAR2(2000), - "LOCAL_PATH" VARCHAR2(2000) - ); diff --git a/Oracle/pcornet_ontology_updates.csv b/Oracle/pcornet_ontology_updates.csv deleted file mode 100644 index b5c10e2..0000000 --- a/Oracle/pcornet_ontology_updates.csv +++ /dev/null @@ -1,9 +0,0 @@ -"C_HLEVEL","C_FULLNAME","C_NAME","C_SYNONYM_CD","C_VISUALATTRIBUTES","C_TOTALNUM","C_BASECODE","C_METADATAXML","C_FACTTABLECOLUMN","C_TABLENAME","C_COLUMNNAME","C_COLUMNDATATYPE","C_OPERATOR","C_DIMCODE","C_COMMENT","C_TOOLTIP","M_APPLIED_PATH","UPDATE_DATE","DOWNLOAD_DATE","IMPORT_DATE","SOURCESYSTEM_CD","VALUETYPE_CD","M_EXCLUSION_CD","C_PATH","C_SYMBOL","PCORI_BASECODE" -2,"\PCORI\DEMOGRAPHIC\HISPANIC\","Ethnicity","N","CAE",,,"3.22014-05-09T11:10:26.266-04:00EnumNoUnknownYes ","concept_cd","CONCEPT_DIMENSION","concept_path","T","like","\PCORI\DEMOGRAPHIC\HISPANIC\",,"Ethnicity","@","2014-05-09 11:10:28","2014-05-09 11:10:28","2014-05-09 11:10:28","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\","HISPANIC", -3,"\PCORI\DEMOGRAPHIC\HISPANIC\NI\","No information","N","LAE",,"ETHNICITY:NI",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","N","IS","NULL",,"No information: A data field is present in the source system, but the source value is null or blank.","@","2014-06-02 19:40:30","2014-06-02 19:40:30","2014-06-02 19:40:30","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","NI","NI" -3,"\PCORI\DEMOGRAPHIC\HISPANIC\N\","Non-Hispanic","N","LIE",,"ETHNICITY:NOTHISPANIC",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'N'",,"Non-Hispanic","@","2014-05-09 11:12:04","2014-05-09 11:12:04","2014-05-09 11:12:04","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","N","N" -3,"\PCORI\DEMOGRAPHIC\HISPANIC\OT\","Other","N","LAE",,"ETHNICITY:OT",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'OT'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2014-06-02 19:44:16","2014-06-02 19:44:16","2014-06-02 19:44:16","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","OT","OT" -3,"\PCORI\DEMOGRAPHIC\HISPANIC\UN\","Unknown","N","LAE",,"ETHNICITY:UN",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'UN','u','unk'",,"Unknown: A data field is present in the source system, but the source value explicitly denotes an unknown value.","@","2014-06-02 19:43:08","2014-06-02 19:43:08","2014-06-02 19:43:08","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","UN","UN" -3,"\PCORI\DEMOGRAPHIC\HISPANIC\Y\","Hispanic","N","LAE",,"ETHNICITY:HISPANIC",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'Y','hib','his/black','his/white','hiw','hispanic'",,"A person of Cuban, Mexican, Puerto Rican, South or Central American, or other Spanish culture or origin, regardless of race. \ Hisptanic","@","2014-05-09 15:21:33","2014-05-09 11:11:12","2014-05-09 11:11:12","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","Y","Y" -3,"\PCORI\DEMOGRAPHIC\RACE\NI\","No information","N","LAE",,"RACE:NI",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","N","IS","'@'",,"No information: A data field is present in the source system, but the source value is null or blank.","@","2016-03-30 16:48:29","2016-03-30 16:48:29","2016-03-30 16:48:29","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","NI","NI" -3,"\PCORI\DEMOGRAPHIC\RACE\OT\","Other","N","LAE",,"RACE:OT",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","T","IN","'OT','o','d','deferred'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2016-03-30 16:48:29","2016-03-30 16:48:29","2016-03-30 16:48:29","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","OT","OT" diff --git a/Oracle/pcornet_ontology_updates.ctl b/Oracle/pcornet_ontology_updates.ctl deleted file mode 100644 index c0d966d..0000000 --- a/Oracle/pcornet_ontology_updates.ctl +++ /dev/null @@ -1,32 +0,0 @@ -options (direct=true, errors=0, skip=1) -load data -truncate into table PCORNET_ONTOLOGY_UPDATES -fields terminated by ',' optionally enclosed by '"' -trailing nullcols( - C_HLEVEL, - C_FULLNAME, - C_NAME CHAR(2000), - C_SYNONYM_CD, - C_VISUALATTRIBUTES, - C_TOTALNUM, - C_BASECODE, - C_METADATAXML CHAR(100000), - C_FACTTABLECOLUMN, - C_TABLENAME, - C_COLUMNNAME, - C_COLUMNDATATYPE, - C_OPERATOR, - C_DIMCODE CHAR(700), - C_COMMENT, - C_TOOLTIP CHAR(900), - M_APPLIED_PATH, - UPDATE_DATE DATE 'YYYY-MM-DD HH24:MI:SS', - DOWNLOAD_DATE DATE 'YYYY-MM-DD HH24:MI:SS', - IMPORT_DATE DATE 'YYYY-MM-DD HH24:MI:SS', - SOURCESYSTEM_CD, - VALUETYPE_CD, - M_EXCLUSION_CD, - C_PATH, - C_SYMBOL, - PCORI_BASECODE - ) \ No newline at end of file diff --git a/Oracle/pcornet_ontology_updates_ddl.sql b/Oracle/pcornet_ontology_updates_ddl.sql deleted file mode 100644 index 82ec392..0000000 --- a/Oracle/pcornet_ontology_updates_ddl.sql +++ /dev/null @@ -1,29 +0,0 @@ -CREATE TABLE pcornet_ontology_updates ( - "C_HLEVEL" NUMBER(22,0) NOT NULL, - "C_FULLNAME" VARCHAR2(700) NOT NULL, - "C_NAME" VARCHAR2(2000) NOT NULL, - "C_SYNONYM_CD" CHAR(1) NOT NULL, - "C_VISUALATTRIBUTES" CHAR(3) NOT NULL, - "C_TOTALNUM" NUMBER(22,0) NULL, - "C_BASECODE" VARCHAR2(50) NULL, - "C_METADATAXML" CLOB NULL, - "C_FACTTABLECOLUMN" VARCHAR2(50) NOT NULL, - "C_TABLENAME" VARCHAR2(50) NOT NULL, - "C_COLUMNNAME" VARCHAR2(50) NOT NULL, - "C_COLUMNDATATYPE" VARCHAR2(50) NOT NULL, - "C_OPERATOR" VARCHAR2(10) NOT NULL, - "C_DIMCODE" VARCHAR2(700) NOT NULL, - "C_COMMENT" CLOB NULL, - "C_TOOLTIP" VARCHAR2(900) NULL, - "M_APPLIED_PATH" VARCHAR2(700) NOT NULL, - "UPDATE_DATE" DATE NOT NULL, - "DOWNLOAD_DATE" DATE NULL, - "IMPORT_DATE" DATE NULL, - "SOURCESYSTEM_CD" VARCHAR2(50) NULL, - "VALUETYPE_CD" VARCHAR2(50) NULL, - "M_EXCLUSION_CD" VARCHAR2(25) NULL, - "C_PATH" VARCHAR2(700) NULL, - "C_SYMBOL" VARCHAR2(50) NULL, - "PCORI_BASECODE" VARCHAR2(50) NULL -) -; From 511aa15561a34973e43fe2f57928e04fa3158a9c Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 6 Jun 2017 12:37:35 -0500 Subject: [PATCH 273/507] Fixed errors in LogTaskComplete logging calls in PCORNetLoader --- Oracle/PCORNetLoader_ora.sql | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 88b360b..d0bc0f2 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1129,7 +1129,7 @@ begin if start_with in ('PCORNetDemographic', 'PCORNetEncounter') then LogTaskStart(release_name, 'PCORNetEncounter', build_num, data_source); PCORNetEncounter; - LogTaskComplete(release_name, 'PCORNetEncounter', build_num, data_source); + LogTaskComplete(release_name, 'PCORNetEncounter', build_num, 'ENCOUNTER'); end if; if start_with in ( @@ -1137,7 +1137,7 @@ begin ) then LogTaskStart(release_name, 'PCORNetDiagnosis', build_num, data_source); PCORNetDiagnosis; - LogTaskComplete(release_name, 'PCORNetDiagnosis', build_num, data_source); + LogTaskComplete(release_name, 'PCORNetDiagnosis', build_num, 'DIAGNOSIS'); end if; if start_with in ( @@ -1146,7 +1146,7 @@ begin ) then LogTaskStart(release_name, 'PCORNetCondition', build_num, data_source); PCORNetCondition; - LogTaskComplete(release_name, 'PCORNetCondition', build_num, data_source); + LogTaskComplete(release_name, 'PCORNetCondition', build_num, 'CONDITION'); end if; if start_with in ( @@ -1155,7 +1155,7 @@ begin ) then LogTaskStart(release_name, 'PCORNetProcedure', build_num, data_source); PCORNetProcedure; - LogTaskComplete(release_name, 'PCORNetProcedure', build_num, data_source); + LogTaskComplete(release_name, 'PCORNetProcedure', build_num, 'PROCEDURE'); end if; if start_with in ( @@ -1164,7 +1164,7 @@ begin ) then LogTaskStart(release_name, 'PCORNetVital', build_num, data_source); PCORNetVital; - LogTaskComplete(release_name, 'PCORNetVital', build_num, data_source); + LogTaskComplete(release_name, 'PCORNetVital', build_num, 'VITAL'); end if; if start_with in ( @@ -1173,7 +1173,7 @@ begin ) then LogTaskStart(release_name, 'PCORNetEnroll', build_num, data_source); PCORNetEnroll; - LogTaskComplete(release_name, 'PCORNetEnroll', build_num, data_source); + LogTaskComplete(release_name, 'PCORNetEnroll', build_num, 'ENROLLMENT'); end if; if start_with in ( @@ -1183,7 +1183,7 @@ begin ) then LogTaskStart(release_name, 'PCORNetLabResultCM', build_num, data_source); PCORNetLabResultCM; - LogTaskComplete(release_name, 'PCORNetLabResultCM', build_num, data_source); + LogTaskComplete(release_name, 'PCORNetLabResultCM', build_num, 'LAB_RESULT_CM'); end if; if start_with in ( @@ -1193,7 +1193,7 @@ begin ) then LogTaskStart(release_name, 'PCORNetPrescribing', build_num, data_source); PCORNetPrescribing; - LogTaskComplete(release_name, 'PCORNetPrescribing', build_num, data_source); + LogTaskComplete(release_name, 'PCORNetPrescribing', build_num, 'PRESCRIBING'); end if; if start_with in ( @@ -1203,7 +1203,7 @@ begin ) then LogTaskStart(release_name, 'PCORNetDispensing', build_num, data_source); PCORNetDispensing; - LogTaskComplete(release_name, 'PCORNetDispensing', build_num, data_source); + LogTaskComplete(release_name, 'PCORNetDispensing', build_num, 'DISPENSING'); end if; if start_with in ( @@ -1214,7 +1214,7 @@ begin ) then LogTaskStart(release_name, 'PCORNetDeath', build_num, data_source); PCORNetDeath; - LogTaskComplete(release_name, 'PCORNetDeath', build_num, data_source); + LogTaskComplete(release_name, 'PCORNetDeath', build_num, 'DEATH'); end if; if start_with in ( @@ -1225,7 +1225,7 @@ begin ) then LogTaskStart(release_name, 'PCORNetHarvest', build_num, data_source); PCORNetHarvest; - LogTaskComplete(release_name, 'PCORNetHarvest', build_num, data_source); + LogTaskComplete(release_name, 'PCORNetHarvest', build_num, 'HARVEST'); end if; end PCORNetLoader; / From 1f4a85fc537745a0f3144613ee5c524c1ba4bb1e Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 6 Jun 2017 13:43:06 -0500 Subject: [PATCH 274/507] Removed CSV upload now handled in h2p-mapping --- Oracle/run-i2p-transform.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index be3772d..288cc34 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -3,7 +3,6 @@ set -e python load_csv.py harvest_local harvest_local.csv harvest_local.ctl pcornet_cdm_user pcornet_cdm python load_csv.py PMN_LabNormal pmn_labnormal.csv pmn_labnormal.ctl pcornet_cdm_user pcornet_cdm -python load_csv.py lab_loinc_mapping lab_loinc_mapping.csv lab_loinc_mapping.ctl pcornet_cdm_user pcornet_cdm # Run some tests sqlplus /nolog < Date: Tue, 6 Jun 2017 15:55:20 -0500 Subject: [PATCH 275/507] Fixed SQL quoting and insert into syntax --- Oracle/PCORNetLoader_ora.sql | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index fdd90a8..219e419 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -203,7 +203,7 @@ select * from (select distinct f.patient_num,encounter_num,SUBSTR(c_fullname,22,2) drg_type,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,3) drg from i2b2fact f inner join demographic d on f.patient_num=d.patid inner join pcornet_enc enc on enc.c_basecode = f.concept_cd -and enc.c_fullname like ''\PCORI\ENCOUNTER\DRG\%'') drg1 group by patient_num,encounter_num,drg_type) drg) drg +and enc.c_fullname like '\PCORI\ENCOUNTER\DRG\%') drg1 group by patient_num,encounter_num,drg_type) drg) drg where rn=1; execute immediate 'create index drg_idx on drg (patient_num, encounter_num)'; @@ -232,7 +232,7 @@ left outer join drg -- This section is bugfixed to only include 1 drg if multipl left outer join -- Encounter type. Note that this requires a full table scan on the ontology table, so it is not particularly efficient. (select patient_num, encounter_num, inout_cd,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,2) pcori_enctype from i2b2visit v - inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype + inner join pcornet_enc e on c_dimcode like '%'||inout_cd||'%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num; execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; @@ -254,7 +254,7 @@ execute immediate 'truncate table diagnosis'; execute immediate 'truncate table sourcefact'; execute immediate 'truncate table pdxfact'; -insert into table sourcefact +insert into sourcefact select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname from i2b2fact factline inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num @@ -264,7 +264,7 @@ insert into table sourcefact execute immediate 'create index sourcefact_idx on sourcefact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('SOURCEFACT'); -insert into table pdxfact +insert into pdxfact select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode pdxsource,dxsource.c_fullname from i2b2fact factline inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num @@ -737,7 +737,7 @@ execute immediate 'truncate table quantity'; execute immediate 'truncate table refills'; execute immediate 'truncate table supply'; -insert into table basis +insert into basis select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact basis inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num join pcornet_med basiscode @@ -747,7 +747,7 @@ select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd, execute immediate 'create unique index basis_idx on basis (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('BASIS'); -insert into table freq +insert into freq select pcori_basecode,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact freq inner join encounter enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num join pcornet_med freqcode @@ -757,7 +757,7 @@ select pcori_basecode,instance_num,start_date,provider_id,concept_cd,encounter_n execute immediate 'create unique index freq_idx on freq (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('FREQ'); -insert into table quantity +insert into quantity select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact quantity inner join encounter enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num join pcornet_med quantitycode @@ -767,7 +767,7 @@ select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,mod execute immediate 'create unique index quantity_idx on quantity (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('QUANTITY'); -insert into table refills +insert into refills select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact refills inner join encounter enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num join pcornet_med refillscode @@ -777,7 +777,7 @@ select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,mod execute immediate 'create unique index refills_idx on refills (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('REFILLS'); -insert into table supply +insert into supply select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact supply inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num join pcornet_med supplycode From f0b316c28c9dbc4c6cf80d73cd92bf686da9d2ca Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 6 Jun 2017 16:00:54 -0500 Subject: [PATCH 276/507] Removed unclosed parenthesis --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 219e419..f32b346 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -570,7 +570,7 @@ select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, from i2b2fact inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode -where c_fullname LIKE '\PCORI_MOD\PRIORITY\%'); +where c_fullname LIKE '\PCORI_MOD\PRIORITY\%'; execute immediate 'create index priority_idx on priority (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('PRIORITY'); From 6fa57ef757c20594db3201d08c33aa032cc3fbeb Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 7 Jun 2017 08:33:41 -0500 Subject: [PATCH 277/507] Missed removing one of the old drop table statements --- Oracle/PCORNetLoader_ora.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index f32b346..dc60391 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -357,8 +357,6 @@ PMN_DROPSQL('drop index sourcefact2_idx'); execute immediate 'truncate table condition'; execute immediate 'truncate table sourcefact2'; -PMN_DROPSQL('DROP TABLE sourcefact2'); - insert into sourcefact2 select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname from i2b2fact factline From d3933a53be106a10349065705f8d433b9d431a4f Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 7 Jun 2017 10:02:21 -0500 Subject: [PATCH 278/507] Fixed issue with adding index to the wrong table --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index dc60391..bda5924 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -364,7 +364,7 @@ insert into sourcefact2 inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode where dxsource.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\%'; -execute immediate 'create index sourcefact2_idx on sourcefact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +execute immediate 'create index sourcefact2_idx on sourcefact2 (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('SOURCEFACT2'); create_error_table('CONDITION'); From 4a2eb3bfeb037e601709e727146863d619c14857 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 7 Jun 2017 14:34:09 -0500 Subject: [PATCH 279/507] Fixed typo in logging procedure call --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index bda5924..8047f19 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1171,7 +1171,7 @@ begin ) then LogTaskStart(release_name, 'PCORNetProcedure', build_num, data_source); PCORNetProcedure; - LogTaskComplete(release_name, 'PCORNetProcedure', build_num, 'PROCEDURE'); + LogTaskComplete(release_name, 'PCORNetProcedure', build_num, 'PROCEDURES'); end if; if start_with in ( From e00cd1d8e1232ee3dcde0fd99b0ea195402addd8 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Jun 2017 09:10:30 -0500 Subject: [PATCH 280/507] Field additions and updates required by PCORNet CDM v3.1 --- Oracle/PCORNetInit.sql | 22 +++++++++++++------- Oracle/PCORNetLoader_ora.sql | 40 ++++++++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index 899ba1a..44751ca 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -260,10 +260,14 @@ CREATE TABLE demographic( BIRTH_DATE date NULL, BIRTH_TIME varchar(5) NULL, SEX varchar(2) NULL, + SEXUAL_ORIENTATION varchar(2) NULL, + GENDER_IDENTITY varchar(2) NULL, HISPANIC varchar(2) NULL, BIOBANK_FLAG varchar(1) DEFAULT 'N', RACE varchar(2) NULL, RAW_SEX varchar(50) NULL, + RAW_SEXUAL_ORIENTATION varchar(50) NULL, + RAW_GENDER_IDENTITY varchar(50) NULL, RAW_HISPANIC varchar(50) NULL, RAW_RACE varchar(50) NULL ) @@ -355,6 +359,7 @@ CREATE TABLE diagnosis( DX varchar(18) NOT NULL, DX_TYPE varchar(2) NOT NULL, DX_SOURCE varchar(2) NOT NULL, + DX_ORIGIN varchar(2) NULL, PDX varchar(2) NULL, RAW_DX varchar(50) NULL, RAW_DX_TYPE varchar(50) NULL, @@ -713,14 +718,17 @@ CREATE TABLE prescribing( RX_ORDER_TIME varchar (5) NULL, RX_START_DATE date NULL, RX_END_DATE date NULL, - RX_QUANTITY int NULL, - RX_REFILLS int NULL, - RX_DAYS_SUPPLY int NULL, + RX_QUANTITY number(18,5) NULL, + RX_QUANTITY_UNIT varchar(2) NULL, + RX_REFILLS number(18,5) NULL, + RX_DAYS_SUPPLY number (18,5) NULL, RX_FREQUENCY varchar(2) NULL, RX_BASIS varchar (2) NULL, - RXNORM_CUI int NULL, + RXNORM_CUI varchar(8) NULL, RAW_RX_MED_NAME varchar (50) NULL, RAW_RX_FREQUENCY varchar (50) NULL, + RAW_RX_QUANTITY varchar(50) NULL, + RAW_RX_NDC varchar(50) NULL, RAW_RXNORM_CUI varchar (50) NULL ) / @@ -836,9 +844,9 @@ CREATE TABLE dispensing( PRESCRIBINGID varchar(19) NULL, DISPENSE_DATE date NOT NULL, NDC varchar (11) NOT NULL, - DISPENSE_SUP int, - DISPENSE_AMT int, - RAW_NDC varchar (50) + DISPENSE_SUP number(18) NULL, + DISPENSE_AMT number(18) NULL, + RAW_NDC varchar (50) NULL ) / diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 8047f19..4b160d0 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -27,11 +27,13 @@ create or replace procedure PCORNetDemographic as sqltext varchar2(4000); cursor getsql is --1 -- S,R,NH - select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''1'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| ''''||sex.pcori_basecode||''','|| + '''NI'','|| + '''NI'','|| '''NI'','|| ''''||race.pcori_basecode||''''|| ' from i2b2patient p '|| @@ -44,11 +46,13 @@ cursor getsql is and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' and sex.c_visualattributes like 'L%' union -- A - S,R,H -select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| +select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''A'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| ''''||sex.pcori_basecode||''','|| + '''NI'','|| + '''NI'','|| ''''||hisp.pcori_basecode||''','|| ''''||race.pcori_basecode||''''|| ' from i2b2patient p '|| @@ -63,11 +67,13 @@ select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPA and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' and sex.c_visualattributes like 'L%' union --2 S, nR, nH - select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''2'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| ''''||sex.pcori_basecode||''','|| + '''NI'','|| + '''NI'','|| '''NI'','|| '''NI'''|| ' from i2b2patient p '|| @@ -78,11 +84,13 @@ union --2 S, nR, nH where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' and sex.c_visualattributes like 'L%' union --3 -- nS,R, NH - select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''3'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| '''NI'','|| + '''NI'','|| + '''NI'','|| '''NI'','|| ''''||race.pcori_basecode||''''|| ' from i2b2patient p '|| @@ -93,11 +101,13 @@ union --3 -- nS,R, NH where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' and race.c_visualattributes like 'L%' union --B -- nS,R, H - select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''B'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| '''NI'','|| + '''NI'','|| + '''NI'','|| ''''||hisp.pcori_basecode||''','|| ''''||race.pcori_basecode||''''|| ' from i2b2patient p '|| @@ -110,11 +120,13 @@ union --B -- nS,R, H and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' and hisp.c_visualattributes like 'L%' union --4 -- S, NR, H - select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''4'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| ''''||sex.pcori_basecode||''','|| + '''NI'','|| + '''NI'','|| ''''||hisp.pcori_basecode||''','|| '''NI'''|| ' from i2b2patient p '|| @@ -127,11 +139,13 @@ union --4 -- S, NR, H and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' and hisp.c_visualattributes like 'L%' union --5 -- NS, NR, H - select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''5'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| '''NI'','|| + '''NI'','|| + '''NI'','|| ''''||hisp.pcori_basecode||''','|| '''NI'''|| ' from i2b2patient p '|| @@ -142,11 +156,13 @@ union --5 -- NS, NR, H where hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' and hisp.c_visualattributes like 'L%' union --6 -- NS, NR, nH - select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, HISPANIC, RACE) '|| + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''6'',patient_num, '|| ' birth_date, '|| ' to_char(birth_date,''HH:MI''), '|| '''NI'','|| + '''NI'','|| + '''NI'','|| '''NI'','|| '''NI'''|| ' from i2b2patient p '|| @@ -274,7 +290,7 @@ insert into pdxfact execute immediate 'create index pdxfact_idx on pdxfact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('PDXFACT'); -insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, pdx) +insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, dx_origin, pdx) /* KUMC started billing with ICD10 on Oct 1, 2015. */ with icd10_transition as ( select date '2015-10-01' as cutoff from dual @@ -317,6 +333,7 @@ select distinct factline.patient_num, factline.encounter_num encounterid, enc_ty else '09' end dxtype, CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END dx_source, + 'BI' dx_origin, CASE WHEN enc_type in ('EI', 'IP', 'IS') -- PDX is "relevant only on IP and IS encounters" THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, ':')+1,2),'NI') ELSE 'X' END PDX @@ -617,7 +634,7 @@ INSERT INTO lab_result_cm SELECT DISTINCT M.patient_num patid, M.encounter_num encounterid, -CASE WHEN ont_parent.C_BASECODE LIKE 'LAB_NAME%' then SUBSTR (ont_parent.c_basecode,10, 10) ELSE null END LAB_NAME, +CASE WHEN ont_parent.C_BASECODE LIKE 'LAB_NAME%' then SUBSTR (ont_parent.c_basecode,10, 10) ELSE 'UN' END LAB_NAME, CASE WHEN lab.pcori_specimen_source like '%or SR_PLS' THEN 'SR_PLS' WHEN lab.pcori_specimen_source is null then 'NI' ELSE lab.pcori_specimen_source END specimen_source, -- (Better way would be to fix the column in the ontology but this will work) NVL(lab.pcori_basecode, 'NI') LAB_LOINC, NVL(p.PRIORITY,'NI') PRIORITY, @@ -796,6 +813,7 @@ insert into prescribing ( ,RX_END_DATE ,RXNORM_CUI --using pcornet_med pcori_cui - new column! ,RX_QUANTITY ---- modifier nval_num + ,RX_QUANTITY_UNIT ,RX_REFILLS -- modifier nval_num ,RX_DAYS_SUPPLY -- modifier nval_num ,RX_FREQUENCY --modifier with basecode lookup @@ -805,7 +823,7 @@ insert into prescribing ( ,RAW_RXNORM_CUI ) select distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH:MI'), m.start_date start_date, m.end_date, mo.pcori_cui - ,quantity.nval_num quantity, refills.nval_num refills, supply.nval_num supply, substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) frequency, + ,quantity.nval_num quantity, 'NI' rx_quantity_unit, refills.nval_num refills, supply.nval_num supply, substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) frequency, substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) basis , substr(mo.c_name, 1, 50) raw_rx_med_name, substr(mo.c_basecode, 1, 50) raw_rxnorm_cui from i2b2medfact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode From 5eac271a45db4657817f463ae3667bbf9f636e9c Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Jun 2017 16:12:13 -0500 Subject: [PATCH 281/507] Fixed quoting error and improved indexes --- Oracle/PCORNetLoader_ora.sql | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 8047f19..0a358fd 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -158,7 +158,7 @@ union --6 -- NS, NR, nH begin pcornet_popcodelist; -PMN_DROPSQL('drop index demographic_idx'); +PMN_DROPSQL('drop index demographic_pk'); execute immediate 'truncate table demographic'; @@ -172,7 +172,7 @@ FETCH getsql INTO sqltext; END LOOP; CLOSE getsql; -execute immediate 'create index demographic_idx on demographic (PATID)'; +execute immediate 'create unique index demographic_pk on demographic (PATID)'; GATHER_TABLE_STATS('DEMOGRAPHIC'); end PCORNetDemographic; @@ -190,7 +190,8 @@ ORA-00904: "LOCATION_ZIP": invalid identifier create or replace procedure PCORNetEncounter as begin -PMN_DROPSQL('drop index encounter_idx'); +PMN_DROPSQL('drop index encounter_pk'); +PMN_DROPSQL('drop index encounter_idx') PMN_DROPSQL('drop index drg_idx'); execute immediate 'truncate table encounter'; @@ -232,9 +233,10 @@ left outer join drg -- This section is bugfixed to only include 1 drg if multipl left outer join -- Encounter type. Note that this requires a full table scan on the ontology table, so it is not particularly efficient. (select patient_num, encounter_num, inout_cd,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,2) pcori_enctype from i2b2visit v - inner join pcornet_enc e on c_dimcode like '%'||inout_cd||'%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype + inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num; +execute immediate 'create unique index encounter_pk on encounter (ENCOUNTERID)'; execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; GATHER_TABLE_STATS('ENCOUNTER'); From e27f0790b046cb4367d6785c5fc490e237cca35d Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 8 Jun 2017 16:17:52 -0500 Subject: [PATCH 282/507] Added missing semicolon --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 0a358fd..fb163ba 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -191,7 +191,7 @@ create or replace procedure PCORNetEncounter as begin PMN_DROPSQL('drop index encounter_pk'); -PMN_DROPSQL('drop index encounter_idx') +PMN_DROPSQL('drop index encounter_idx'); PMN_DROPSQL('drop index drg_idx'); execute immediate 'truncate table encounter'; From 76d9be6ddb331f0452b3ccb177f60fd4be6e4c92 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 12 Jun 2017 08:27:23 -0500 Subject: [PATCH 283/507] Mislabeled index on procedures --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index fb163ba..77762a2 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -417,7 +417,7 @@ from i2b2fact fact inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd where pr.c_fullname like '\PCORI\PROCEDURE\%'; -execute immediate 'create index procedures_patid on procedures (PATID, ENCOUNTERID)'; +execute immediate 'create index procedures_idx on procedures (PATID, ENCOUNTERID)'; GATHER_TABLE_STATS('PROCEDURES'); end PCORNetProcedure; From 3cddd765fa5c9d8224854ad91d193b4c7b448a64 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 12 Jun 2017 12:18:35 -0500 Subject: [PATCH 284/507] Update PCORNetLoader procedure to make it more readable and easier to update --- Oracle/PCORNetLoader_ora.sql | 89 ++++++++++++++++-------------------- Oracle/run-i2p-transform.sh | 3 +- 2 files changed, 40 insertions(+), 52 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 8d6b9ba..03f4786 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1134,92 +1134,81 @@ end PCORNetPostProc; / -create or replace PROCEDURE PCORNetLoader(start_with VARCHAR2) AS +-------------------------------------------------------------------------------- +-- PCORNetLoader procedure +-- +-- This procedure orchestrates the execution of the procedures defined above, +-- and consists of 13 steps. Using the start_with_step parameter a step number +-- can be provided to start at any point in the sequence. This is helpful when +-- an issue is encountered during execution and restart from the beginning is +-- undesirable. +-- +-- Steps: +-- 1 - PCORNetDemographic +-- 2 - PCORNetEncounter +-- 3 - PCORNetDiagnosis +-- 4 - PCORNetCondition +-- 5 - PCORNetProcedure +-- 6 - PCORNetVital +-- 7 - PCORNetEnroll +-- 8 - PCORNetLabResultCM +-- 9 - PCORNetPrescribing +-- 10 - PCORNetDispensing +-- 11 - PCORNetDeath +-- 12 - PCORNetHarvest +-- 13 - PCORNetPostProc +-- +-------------------------------------------------------------------------------- +create or replace PROCEDURE PCORNetLoader(start_with_step VARCHAR2) AS begin - if start_with in ('PCORNetDemographic') then + if start_with_step = 1 then PCORNetDemographic; end if; - if start_with in ('PCORNetDemographic', 'PCORNetEncounter') then + if start_with_step >= 2 then PCORNetEncounter; end if; - if start_with in ( - 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis' - ) then + if start_with_step >= 3 then PCORNetDiagnosis; end if; - if start_with in ( - 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', - 'PCORNetCondition' - ) then + if start_with_step >= 4 then PCORNetCondition; end if; - if start_with in ( - 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', - 'PCORNetCondition', 'PCORNetProcedure' - ) then + if start_with_step >= 5 then PCORNetProcedure; end if; - if start_with in ( - 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', - 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital' - ) then + if start_with_step >= 6 then PCORNetVital; end if; - if start_with in ( - 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', - 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll' - ) then + if start_with_step >= 7 then PCORNetEnroll; end if; - if start_with in ( - 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', - 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', - 'PCORNetLabResultCM' - ) then + if start_with _step >= 8 then PCORNetLabResultCM; end if; - if start_with in ( - 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', - 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', - 'PCORNetLabResultCM', 'PCORNetPrescribing' - ) then + if start_with_step >= 9 then PCORNetPrescribing; end if; - if start_with in ( - 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', - 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', - 'PCORNetLabResultCM', 'PCORNetPrescribing', 'PCORNetDispensing' - ) then + if start_with_step >= 10 then PCORNetDispensing; end if; - if start_with in ( - 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', - 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', - 'PCORNetLabResultCM', 'PCORNetPrescribing', 'PCORNetDispensing', - 'PCORNetDeath' - ) then + if start_with_step >= 11 then PCORNetDeath; end if; - if start_with in ( - 'PCORNetDemographic', 'PCORNetEncounter', 'PCORNetDiagnosis', - 'PCORNetCondition', 'PCORNetProcedure', 'PCORNetVital', 'PCORNetEnroll', - 'PCORNetLabResultCM', 'PCORNetPrescribing', 'PCORNetDispensing', - 'PCORNetDeath', 'PCORNetHarvest' - ) then + if start_with_step >= 12 then PCORNetHarvest; end if; PCORNetPostProc; end PCORNetLoader; -/ +/ \ No newline at end of file diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 730cf1c..5f6ebf6 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -63,7 +63,6 @@ define min_pat_list_date_dd_mon_rrrr=${min_pat_list_date_dd_mon_rrrr} define min_visit_date_dd_mon_rrrr=${min_visit_date_dd_mon_rrrr} define enrollment_months_back=${enrollment_months_back} define pcornet_cdm_user=${pcornet_cdm_user} -define start_with=${start_with} -- Prepare for transform start PCORNetInit.sql @@ -117,7 +116,7 @@ set linesize 3000; set pagesize 5000; -- Run transform -execute PCORNetLoader('${start_with}'); +execute PCORNetLoader('${start_with_step}'); EOF fi From ace702fa417ad2da8aeb7813acae9fc8485ad6eb Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 12 Jun 2017 12:21:14 -0500 Subject: [PATCH 285/507] Added header comments to PCORNetInit.sql --- Oracle/PCORNetInit.sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index 899ba1a..6ddda4c 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -1,3 +1,18 @@ +------------------------------------------------------------------------------------------- +-- PCORNetInit Script +-- +-- This script prepares target load and intermidiary transform tables, helper +-- functions and procedure, and synonyms and views of the source i2b2 source +-- tables. +-- +-- This script should be run as the initial step of running the i2p-transform, +-- but has been seperated out from PCORNetLoader_ora.sql so that the former can +-- be run without dropping all tables. +-- +-- Created by: Michael Prittie (mprittie@kumc.edu) +-- Adapted from original PCORNetLoader_ora.sql +-------------------------------------------------------------------------------- + -------------------------------------------------------------------------------- -- HELPER FUNCTIONS AND PROCEDURES From b920a4113e0cca197a86e4f198de58fff0fe8c9f Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 12 Jun 2017 13:05:34 -0500 Subject: [PATCH 286/507] Remove unused PCORNetReport --- Oracle/PCORNetInit.sql | 116 +++++++++++++---------------------- Oracle/PCORNetLoader_ora.sql | 60 ------------------ 2 files changed, 44 insertions(+), 132 deletions(-) diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index 6ddda4c..0e96fb7 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -111,70 +111,6 @@ END; / --------------------------------------------------------------------------------- --- I2B2 SYNONYMS, VIEWS, AND INTERMEDIARY TABLES --------------------------------------------------------------------------------- - - -CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT -/ - -CREATE OR REPLACE SYNONYM I2B2MEDFACT FOR OBSERVATION_FACT_MEDS -/ - -BEGIN -PMN_DROPSQL('DROP TABLE i2b2patient_list'); -END; -/ - -CREATE table i2b2patient_list as -select * from -( -select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') -) where ROWNUM<100000000 -/ - -create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) -/ - -create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') +) where ROWNUM<100000000 +/ + +create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) +/ + +create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE Date: Mon, 12 Jun 2017 13:10:35 -0500 Subject: [PATCH 287/507] Further improvements on PCORNetLoader proc --- Oracle/PCORNetLoader_ora.sql | 21 +++++++++++++++++++++ Oracle/run-i2p-transform.sh | 7 +------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 4c709b0..fc7ef0e 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1100,7 +1100,28 @@ end PCORNetPostProc; -- -------------------------------------------------------------------------------- create or replace PROCEDURE PCORNetLoader(start_with_step VARCHAR2) AS +start_with_step int; begin + + select step into start_with_step from ( + select 1 step, 'PCORNetDemographic' proc from dual union + select 2 step, 'PCORNetEncounter' proc from dual union + select 3 step, 'PCORNetDiagnosis' proc from dual union + select 4 step, 'PCORNetCondition' proc from dual union + select 5 step, 'PCORNetProcedure' proc from dual union + select 6 step, 'PCORNetVital' proc from dual union + select 7 step, 'PCORNetEnroll' proc from dual union + select 8 step, 'PCORNetLabResultCM' proc from dual union + select 9 step, 'PCORNetPrescribing' proc from dual union + select 10 step, 'PCORNetDispensing' proc from dual union + select 11 step, 'PCORNetDeath' proc from dual union + select 12 step, 'PCORNetHarvest' proc from dual union + select 13 step, 'PCORNetPostProc' proc from dual + ) where proc=start_with; + + insert into output + select start_with_step from dual; + if start_with_step = 1 then PCORNetDemographic; end if; diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 5f6ebf6..ad1544e 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -104,9 +104,6 @@ fi ########### Exectute Transform ########### -if [ ${start_with} != 'PCORNetPostProc' ] -then - sqlplus /nolog < Date: Mon, 12 Jun 2017 13:33:12 -0500 Subject: [PATCH 288/507] Removed uneeded statements --- Oracle/PCORNetLoader_ora.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index fc7ef0e..6c672bc 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1119,9 +1119,6 @@ begin select 13 step, 'PCORNetPostProc' proc from dual ) where proc=start_with; - insert into output - select start_with_step from dual; - if start_with_step = 1 then PCORNetDemographic; end if; From 73a1f120ba027feb33e49288152cbb25ab9616ea Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 12 Jun 2017 15:39:01 -0500 Subject: [PATCH 289/507] Error in SQL --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 5088898..c293455 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1179,7 +1179,7 @@ begin LogTaskComplete(release_name, 'PCORNetEnroll', build_num, 'ENROLLMENT'); end if; - if start_with _step >= 8 then + if start_with_step >= 8 then LogTaskStart(release_name, 'PCORNetLabResultCM', build_num, data_source); PCORNetLabResultCM; LogTaskComplete(release_name, 'PCORNetLabResultCM', build_num, 'LAB_RESULT_CM'); From cd488895e1b0b2d21d60b5c8df893de14906fcce Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 12 Jun 2017 15:47:47 -0500 Subject: [PATCH 290/507] Added accidentally removed PCORNetLoader parameters --- Oracle/PCORNetLoader_ora.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index c293455..ae758a2 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1117,7 +1117,9 @@ end PCORNetPostProc; -- 13 - PCORNetPostProc -- -------------------------------------------------------------------------------- -create or replace PROCEDURE PCORNetLoader(start_with_step VARCHAR2) AS +create or replace PROCEDURE PCORNetLoader(start_with VARCHAR2, + release_name VARCHAR2, build_num NUMBER, data_source VARCHAR2 +) AS start_with_step int; begin From 34103549ed02eb211669273a46702a814c00e795 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 12 Jun 2017 16:39:12 -0500 Subject: [PATCH 291/507] Using wrong inequality --- Oracle/PCORNetLoader_ora.sql | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index ae758a2..b97c7b8 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1145,67 +1145,67 @@ begin LogTaskComplete(release_name, 'PCORNetDemographic', build_num, 'DEMOGRAPHIC'); end if; - if start_with_step >= 2 then + if start_with_step <= 2 then LogTaskStart(release_name, 'PCORNetEncounter', build_num, data_source); PCORNetEncounter; LogTaskComplete(release_name, 'PCORNetEncounter', build_num, 'ENCOUNTER'); end if; - if start_with_step >= 3 then + if start_with_step <= 3 then LogTaskStart(release_name, 'PCORNetDiagnosis', build_num, data_source); PCORNetDiagnosis; LogTaskComplete(release_name, 'PCORNetDiagnosis', build_num, 'DIAGNOSIS'); end if; - if start_with_step >= 4 then + if start_with_step <= 4 then LogTaskStart(release_name, 'PCORNetCondition', build_num, data_source); PCORNetCondition; LogTaskComplete(release_name, 'PCORNetCondition', build_num, 'CONDITION'); end if; - if start_with_step >= 5 then + if start_with_step <= 5 then LogTaskStart(release_name, 'PCORNetProcedure', build_num, data_source); PCORNetProcedure; LogTaskComplete(release_name, 'PCORNetProcedure', build_num, 'PROCEDURES'); end if; - if start_with_step >= 6 then + if start_with_step <= 6 then LogTaskStart(release_name, 'PCORNetVital', build_num, data_source); PCORNetVital; LogTaskComplete(release_name, 'PCORNetVital', build_num, 'VITAL'); end if; - if start_with_step >= 7 then + if start_with_step <= 7 then LogTaskStart(release_name, 'PCORNetEnroll', build_num, data_source); PCORNetEnroll; LogTaskComplete(release_name, 'PCORNetEnroll', build_num, 'ENROLLMENT'); end if; - if start_with_step >= 8 then + if start_with_step <= 8 then LogTaskStart(release_name, 'PCORNetLabResultCM', build_num, data_source); PCORNetLabResultCM; LogTaskComplete(release_name, 'PCORNetLabResultCM', build_num, 'LAB_RESULT_CM'); end if; - if start_with_step >= 9 then + if start_with_step <= 9 then LogTaskStart(release_name, 'PCORNetPrescribing', build_num, data_source); PCORNetPrescribing; LogTaskComplete(release_name, 'PCORNetPrescribing', build_num, 'PRESCRIBING'); end if; - if start_with_step >= 10 then + if start_with_step <= 10 then LogTaskStart(release_name, 'PCORNetDispensing', build_num, data_source); PCORNetDispensing; LogTaskComplete(release_name, 'PCORNetDispensing', build_num, 'DISPENSING'); end if; - if start_with_step >= 11 then + if start_with_step <= 11 then LogTaskStart(release_name, 'PCORNetDeath', build_num, data_source); PCORNetDeath; LogTaskComplete(release_name, 'PCORNetDeath', build_num, 'DEATH'); end if; - if start_with_step >= 12 then + if start_with_step <= 12 then LogTaskStart(release_name, 'PCORNetHarvest', build_num, data_source); PCORNetHarvest; LogTaskComplete(release_name, 'PCORNetHarvest', build_num, 'HARVEST'); From 121f5252c18fccf276b049024671ec987a923436 Mon Sep 17 00:00:00 2001 From: Brennan Connolly Date: Tue, 20 Jun 2017 10:41:03 -0500 Subject: [PATCH 292/507] Port SCILHS Code for incorporating DX_ORIGIN into the transform Includes creation, insertion, truncation, indexing, and join code for the originfact table. --- Oracle/PCORNetInit.sql | 16 ++++++++++++++++ Oracle/PCORNetLoader_ora.sql | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index ad84a19..0269d59 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -403,6 +403,22 @@ CREATE TABLE PDXFACT ( ) / +BEGIN +PMN_DROPSQL('DROP TABLE originfact'); +END; +/ + +CREATE TABLE ORIGINFACT ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + ORIGINSOURCE VARCHAR2(50) NULL, + C_FULLNAME VARCHAR2(700) NOT NULL + ) +/ + -------------------------------------------------------------------------------- -- PROCEDURES -------------------------------------------------------------------------------- diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index b97c7b8..08ad3a7 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -267,10 +267,12 @@ begin PMN_DROPSQL('drop index diagnosis_idx'); PMN_DROPSQL('drop index sourcefact_idx'); PMN_DROPSQL('drop index pdxfact_idx'); +PMN_DROPSQL('drop index originfact_idx'); execute immediate 'truncate table diagnosis'; execute immediate 'truncate table sourcefact'; execute immediate 'truncate table pdxfact'; +execute immediate 'truncate table originfact'; insert into sourcefact select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname @@ -292,6 +294,16 @@ insert into pdxfact execute immediate 'create index pdxfact_idx on pdxfact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('PDXFACT'); +insert into originfact --CDM 3.1 addition + select patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode originsource, dxsource.c_fullname + from i2b2fact factline + inner join pmnENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + and dxsource.c_fullname like '\PCORI_MOD\DX_ORIGIN\%'; + +execute immediate 'create index originfact_idx on originfact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('ORIGINFACT'); + insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, dx_origin, pdx) /* KUMC started billing with ICD10 on Oct 1, 2015. */ with icd10_transition as ( @@ -353,6 +365,12 @@ and factline.encounter_num=pdxfact.encounter_num and factline.provider_id=pdxfact.provider_id and factline.concept_cd=pdxfact.concept_cd and factline.start_date=pdxfact.start_Date +left outer join originfact --CDM 3.1 addition +on factline.patient_num=originfact.patient_num +and factline.encounter_num=originfact.encounter_num +and factline.provider_id=originfact.provider_id +and factline.concept_cd=originfact.concept_cd +and factline.start_date=originfact.start_Date inner join diag on diag.c_basecode = factline.concept_cd cross join icd10_transition where (sourcefact.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\%' or sourcefact.c_fullname is null) From e86d071b967ae1d4c83cfdffb9de24f3514c7564 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 21 Jun 2017 08:30:23 -0500 Subject: [PATCH 293/507] Re-ordered inner join for better execution plan --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 08ad3a7..13a8533 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -448,8 +448,8 @@ select distinct fact.patient_num, enc.encounterid, enc.enc_type, enc.admit_date -- All are billing for now - see https://informatics.gpcnetwork.org/trac/Project/ticket/491 'BI' px_source from i2b2fact fact + inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd inner join encounter enc on enc.patid = fact.patient_num and enc.encounterid = fact.encounter_Num - inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd where pr.c_fullname like '\PCORI\PROCEDURE\%'; execute immediate 'create index procedures_idx on procedures (PATID, ENCOUNTERID)'; From 067ae8c3ed2a417bb9dac346bd16ca0aebf65008 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 21 Jun 2017 08:31:57 -0500 Subject: [PATCH 294/507] Corrected table name --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 13a8533..53f0673 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -297,7 +297,7 @@ GATHER_TABLE_STATS('PDXFACT'); insert into originfact --CDM 3.1 addition select patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode originsource, dxsource.c_fullname from i2b2fact factline - inner join pmnENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join ENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode and dxsource.c_fullname like '\PCORI_MOD\DX_ORIGIN\%'; From 896f8e92eb2514fb549bd41948ec5793deef7f8c Mon Sep 17 00:00:00 2001 From: Brennan Connolly Date: Thu, 22 Jun 2017 10:35:48 -0500 Subject: [PATCH 295/507] Adjust DX_ORIGIN insertion Change from always inserting 'BI' to referencing the originfact table for the insert. --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 53f0673..2549e1b 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -347,7 +347,7 @@ select distinct factline.patient_num, factline.encounter_num encounterid, enc_ty else '09' end dxtype, CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END dx_source, - 'BI' dx_origin, + nvl(SUBSTR(originsource,INSTR(originsource, ':')+1,2),'NI') dx_origin, CASE WHEN enc_type in ('EI', 'IP', 'IS') -- PDX is "relevant only on IP and IS encounters" THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, ':')+1,2),'NI') ELSE 'X' END PDX From 58f4bbdfc6715ea59f6b02258408f321e5a04590 Mon Sep 17 00:00:00 2001 From: Matt Hoag Date: Thu, 22 Jun 2017 10:40:36 -0500 Subject: [PATCH 296/507] Determine `dx_type` based on concept code, not icd10 transition date (#4749) --- Oracle/PCORNetLoader_ora.sql | 37 +++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 53f0673..e51bedf 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -309,6 +309,20 @@ insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, with icd10_transition as ( select date '2015-10-01' as cutoff from dual ) + +/* Encoding of DX type from the CDM 3.1 Specification + http://pcornet.org/wp-content/uploads/2016/11/2016-11-15-PCORnet-Common-Data-Model-v3.1_Specification.pdf + */ +, dx_type as ( + select '09' as icd_9_cm + , '10' as icd_10_cm + , '11' as icd_11_cm + , 'SM' as snomed_ct + , 'NI' as no_info + , 'UN' as unknown + , 'OT' as other + from dual) + /* DX_IDs may have mappings to both ICD9 and ICD10 */ , has9 as ( select distinct c_basecode, pcori_basecode icd9_code @@ -322,15 +336,16 @@ with icd10_transition as ( where diag.c_fullname like '\PCORI\DIAGNOSIS\10%' and pcori_basecode is not null ) -, has9_and_10 as ( - select has9.c_basecode, has9.icd9_code, has10.icd10_code - from has9 join has10 on has9.c_basecode = has10.c_basecode -) -, diag as ( -- replace diag rows by 9_and_10 rows and avoid dups - select distinct diag.c_basecode, diag.pcori_basecode, icd9_code, icd10_code - , case when has9_and_10.c_basecode is null then SUBSTR(diag.c_fullname,18,2) else null end dx_type +, diag as ( + select distinct diag.c_basecode, diag.pcori_basecode, has9.icd9_code, has10.icd10_code + , case when diag.pcori_basecode = has9.icd9_code then (select icd_9_cm from dx_type) + when diag.pcori_basecode = has10.icd10_code then (select icd_10_cm from dx_type) + else (select no_info from dx_type) + end dx_type from pcornet_diag diag - left join has9_and_10 on has9_and_10.c_basecode = diag.c_basecode + left join has9 on has9.c_basecode = diag.c_basecode + left join has10 on has10.c_basecode = diag.c_basecode + where diag.c_fullname like '\PCORI\DIAGNOSIS\%' and diag.pcori_basecode is not null ) @@ -341,11 +356,7 @@ select distinct factline.patient_num, factline.encounter_num encounterid, enc_ty when enc.admit_date >= icd10_transition.cutoff then diag.icd10_code else diag.icd9_code end dx - , case - when diag.dx_type is not null then diag.dx_type - when enc.admit_date >= icd10_transition.cutoff then '10' - else '09' - end dxtype, + , diag.dx_type dxtype, CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END dx_source, 'BI' dx_origin, CASE WHEN enc_type in ('EI', 'IP', 'IS') -- PDX is "relevant only on IP and IS encounters" From 9560dec6ca8530c69e593b08c751a374b2c2c4b5 Mon Sep 17 00:00:00 2001 From: Matt Hoag Date: Thu, 22 Jun 2017 13:33:09 -0500 Subject: [PATCH 297/507] Keep the same cardinality for diagnosis facts when converting from i2b2 to pcori. - Old code was joining the fact table with a mapping table that did not have 1 to 1 mappings, which resulted in duplicate facts (one representing the fact as an ICD9 and one representing the fact as an ICD10) - Picks ICD10s over ICD9 after 2015-10 and ICD9s over ICD10 before 2015-10 --- Oracle/PCORNetLoader_ora.sql | 41 ++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index e51bedf..8af6f47 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -349,20 +349,45 @@ with icd10_transition as ( where diag.c_fullname like '\PCORI\DIAGNOSIS\%' and diag.pcori_basecode is not null ) +/* Convert i2b2 diagnosis facts to pcori diagnosis facts + * Note: The mapping of i2b2 diagnosis facts to pcori diagnosis facts + * with respect to ICD9 and ICD10 is not 1-to-1 (there may exist an i2b2 + * diagnosis fact that could be represented as both ICD9 and ICD10 pcori fact) + * + * To prevent an increase in the number of post conversion facts, prioritize + * filtering ICD10s prior to 1/1/2015 and ICD9s after 1/1/2015 if resulting + * facts produce duplicates based on the i2b2 fact primary key. + */ +, diag_fact_merge as ( + select factline.*, diag.* + , row_number() over (partition by factline.patient_num + , factline.encounter_num + , factline.concept_cd + , factline.start_date + , factline.modifier_cd + , factline.instance_num + order by (case when diag.dx_type = (select icd_9_cm from dx_type) + and factline.start_date >= (select cutoff from icd10_transition) then 0 + when diag.dx_type = (select icd_10_cm from dx_type) + and factline.start_date < (select cutoff from icd10_transition) then 0 + else 1 + end) asc) unique_row + from i2b2fact factline + join diag on diag.c_basecode = factline.concept_cd +) +, diag_fact_cutoff_filter as ( + select * from diag_fact_merge where unique_row = 1 +) select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, enc.admit_date, enc.providerid - , case - when diag.pcori_basecode is not null then diag.pcori_basecode - when enc.admit_date >= icd10_transition.cutoff then diag.icd10_code - else diag.icd9_code - end dx - , diag.dx_type dxtype, + , factline.pcori_basecode dx + , factline.dx_type dxtype, CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END dx_source, 'BI' dx_origin, CASE WHEN enc_type in ('EI', 'IP', 'IS') -- PDX is "relevant only on IP and IS encounters" THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, ':')+1,2),'NI') ELSE 'X' END PDX -from i2b2fact factline +from diag_fact_cutoff_filter factline inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num left outer join sourcefact on factline.patient_num=sourcefact.patient_num @@ -382,8 +407,6 @@ and factline.encounter_num=originfact.encounter_num and factline.provider_id=originfact.provider_id and factline.concept_cd=originfact.concept_cd and factline.start_date=originfact.start_Date -inner join diag on diag.c_basecode = factline.concept_cd -cross join icd10_transition where (sourcefact.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\%' or sourcefact.c_fullname is null) -- order by enc.admit_date desc ; From 68786f0afd0719f9bddc704325d1c3be66ad8bef Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 26 Jun 2017 12:43:48 -0500 Subject: [PATCH 298/507] Updated field length for v3.1 --- Oracle/PCORNetInit.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index 0269d59..9e0fc15 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -675,7 +675,7 @@ CREATE TABLE pro_cm( PRO_CM_ID varchar(19) primary key, PATID varchar(50) NOT NULL, ENCOUNTERID varchar(50) NULL, - PRO_ITEM varchar (7) NOT NULL, + PRO_ITEM varchar (20) NOT NULL, PRO_LOINC varchar (10) NULL, PRO_DATE date NOT NULL, PRO_TIME varchar (5) NULL, From 3c7d2c29389467f9893ebe36329b1ef8bdab84cd Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 26 Jun 2017 12:54:23 -0500 Subject: [PATCH 299/507] Update field length in accordance with CDM v3.1 --- Oracle/PCORNetInit.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index 9e0fc15..934c863 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -536,7 +536,7 @@ CREATE TABLE lab_result_cm( RESULT_DATE date NULL, RESULT_TIME varchar(5) NULL, RESULT_QUAL varchar(12) NULL, - RESULT_NUM number (18,5) NULL, + RESULT_NUM number (15,8) NULL, RESULT_MODIFIER varchar(2) NULL, RESULT_UNIT varchar(11) NULL, NORM_RANGE_LOW varchar(10) NULL, From 24f00b2f783481721bfff9d509f361ef420ffa0c Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 27 Jun 2017 09:59:32 -0500 Subject: [PATCH 300/507] Remove bad NDC code if present --- Oracle/PCORNetLoader_ora.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 53f0673..e7d07bb 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1106,6 +1106,12 @@ begin and rx_refills is null ; + /* Removed bad NDC code which make their way in from the source system + (i.e 00000000000 and 99999999999) */ + delete from dispensing + where ndc in ('00000000000', '99999999999') + ; + end PCORNetPostProc; / From ba7964a51b3a5c9aa15b221a2a85103dc742b163 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 27 Jun 2017 10:06:14 -0500 Subject: [PATCH 301/507] _TIME fields were switched to using 24 hour clock --- Oracle/PCORNetLoader_ora.sql | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index e7d07bb..95b98a8 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -30,7 +30,7 @@ cursor getsql is select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''1'',patient_num, '|| ' birth_date, '|| - ' to_char(birth_date,''HH:MI''), '|| + ' to_char(birth_date,''HH24:MI''), '|| ''''||sex.pcori_basecode||''','|| '''NI'','|| '''NI'','|| @@ -49,7 +49,7 @@ union -- A - S,R,H select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''A'',patient_num, '|| ' birth_date, '|| - ' to_char(birth_date,''HH:MI''), '|| + ' to_char(birth_date,''HH24:MI''), '|| ''''||sex.pcori_basecode||''','|| '''NI'','|| '''NI'','|| @@ -70,7 +70,7 @@ union --2 S, nR, nH select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''2'',patient_num, '|| ' birth_date, '|| - ' to_char(birth_date,''HH:MI''), '|| + ' to_char(birth_date,''HH24:MI''), '|| ''''||sex.pcori_basecode||''','|| '''NI'','|| '''NI'','|| @@ -87,7 +87,7 @@ union --3 -- nS,R, NH select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''3'',patient_num, '|| ' birth_date, '|| - ' to_char(birth_date,''HH:MI''), '|| + ' to_char(birth_date,''HH24:MI''), '|| '''NI'','|| '''NI'','|| '''NI'','|| @@ -104,7 +104,7 @@ union --B -- nS,R, H select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''B'',patient_num, '|| ' birth_date, '|| - ' to_char(birth_date,''HH:MI''), '|| + ' to_char(birth_date,''HH24:MI''), '|| '''NI'','|| '''NI'','|| '''NI'','|| @@ -123,7 +123,7 @@ union --4 -- S, NR, H select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''4'',patient_num, '|| ' birth_date, '|| - ' to_char(birth_date,''HH:MI''), '|| + ' to_char(birth_date,''HH24:MI''), '|| ''''||sex.pcori_basecode||''','|| '''NI'','|| '''NI'','|| @@ -142,7 +142,7 @@ union --5 -- NS, NR, H select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''5'',patient_num, '|| ' birth_date, '|| - ' to_char(birth_date,''HH:MI''), '|| + ' to_char(birth_date,''HH24:MI''), '|| '''NI'','|| '''NI'','|| '''NI'','|| @@ -159,7 +159,7 @@ union --6 -- NS, NR, nH select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| ' select ''6'',patient_num, '|| ' birth_date, '|| - ' to_char(birth_date,''HH:MI''), '|| + ' to_char(birth_date,''HH24:MI''), '|| '''NI'','|| '''NI'','|| '''NI'','|| @@ -232,9 +232,9 @@ insert into encounter(PATID,ENCOUNTERID,admit_date ,ADMIT_TIME , DISCHARGE_STATUS ,DRG ,DRG_TYPE ,ADMITTING_SOURCE) select distinct v.patient_num, v.encounter_num, start_Date, - to_char(start_Date,'HH:MI'), + to_char(start_Date,'HH24:MI'), end_Date, - to_char(end_Date,'HH:MI'), + to_char(end_Date,'HH24:MI'), providerid, 'NI' location_zip, /* See TODO above */ (case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, @@ -507,7 +507,7 @@ from ( select obs.patient_num patid, obs.encounter_num encounterid, to_char(obs.start_Date,'YYYY-MM-DD') measure_date, - to_char(obs.start_Date,'HH:MI') measure_time, + to_char(obs.start_Date,'HH24:MI') measure_time, nval_num, pcori_basecode, codes.pcori_code from i2b2fact obs inner join (select c_basecode concept_cd, c_fullname pcori_code, pcori_basecode @@ -663,9 +663,9 @@ NVL(lab.pcori_basecode, 'NI') LAB_PX, 'LC' LAB_PX_TYPE, m.start_date LAB_ORDER_DATE, m.start_date SPECIMEN_DATE, -to_char(m.start_date,'HH:MI') SPECIMEN_TIME, +to_char(m.start_date,'HH24:MI') SPECIMEN_TIME, m.end_date RESULT_DATE, -to_char(m.end_date,'HH:MI') RESULT_TIME, +to_char(m.end_date,'HH24:MI') RESULT_TIME, --CASE WHEN m.ValType_Cd='T' THEN NVL(nullif(m.TVal_Char,''),'NI') ELSE 'NI' END RESULT_QUAL, -- TODO: Should be a standardized value 'NI' RESULT_QUAL, -- Local fix for KUMC (temp) CASE WHEN m.ValType_Cd='N' THEN m.NVAL_NUM ELSE null END RESULT_NUM, @@ -842,7 +842,7 @@ insert into prescribing ( -- ,RAW_RX_FREQUENCY, ,RAW_RXNORM_CUI ) -select distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH:MI'), m.start_date start_date, m.end_date, mo.pcori_cui +select distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH24:MI'), m.start_date start_date, m.end_date, mo.pcori_cui ,quantity.nval_num quantity, 'NI' rx_quantity_unit, refills.nval_num refills, supply.nval_num supply, substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) frequency, substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) basis , substr(mo.c_name, 1, 50) raw_rx_med_name, substr(mo.c_basecode, 1, 50) raw_rxnorm_cui From 7cfe7bee9ecab4769b879f134ce5b66aff6be574 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 28 Jun 2017 09:42:25 -0500 Subject: [PATCH 302/507] Hotfix to remove mapping call in update dimensions script --- Oracle/update_to_cdm_dimensions.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/Oracle/update_to_cdm_dimensions.sh b/Oracle/update_to_cdm_dimensions.sh index 172e449..2e8ca84 100644 --- a/Oracle/update_to_cdm_dimensions.sh +++ b/Oracle/update_to_cdm_dimensions.sh @@ -7,8 +7,6 @@ set -e # export i2b2_data_schema= # export i2b2_meta_schema= -. ./load_pcornet_mapping.sh - sqlplus /nolog < Date: Mon, 10 Jul 2017 09:33:20 -0500 Subject: [PATCH 303/507] Fix to exclude LAB_RESULT_CM.RESULT_NUM values which exceed the spec in length --- Oracle/PCORNetLoader_ora.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 2a436bc..7f91d69 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -749,7 +749,9 @@ and M.concept_cd=l.concept_Cd and M.start_date=l.start_Date WHERE m.ValType_Cd in ('N','T') -and m.MODIFIER_CD='@'; +and m.MODIFIER_CD='@' +and (m.nval_num is null or m.nval_num<10000000 -- exclude lengths that exceed the spec +; execute immediate 'create index lab_result_cm_idx on lab_result_cm (PATID, ENCOUNTERID)'; GATHER_TABLE_STATS('LAB_RESULT_CM'); From 7b1df377bd4b3f5a41d873f5c3e6f9e90fa84208 Mon Sep 17 00:00:00 2001 From: Matt Hoag Date: Mon, 10 Jul 2017 13:59:44 -0500 Subject: [PATCH 304/507] Added Observational Stay `OS` to the encounter_type merge. --- Oracle/heron_encounter_style.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 58e0489..31d5c0e 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -169,17 +169,17 @@ using ( ( max(pcori_code) for pcori_code in ('EI' as ei, 'IP' as ip, 'ED' as ed, 'IS' as e_is - , 'AV' as av, 'OA' as oa, 'OT' as ot + , 'AV' as av, 'OA' as oa, 'OT' as ot, 'OS' as os , 'NI' as ni, 'UN' as un) ) ) /* Single Encounter type to Encounter coercion: IP + ED = EI - EI > IP || ED > IS > AV > OA > OT > NI > UN + EI > IP || ED > OS > IS > AV > OA > OT > NI > UN */ select encounter_num, case when ip is not null and ed is not null then 'EI' - else coalesce(ei, ip, ed, e_is, av, oa, ot, ni, un, 'UN') + else coalesce(ei, ip, ed, os, e_is, av, oa, ot, ni, un, 'UN') end as pcori_code from obs_enc_type_pivot) et on ( vd.encounter_num = et.encounter_num ) From 5cdc71238e8558f44a288b31f158f1642053ccda Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Mon, 10 Jul 2017 15:48:53 -0500 Subject: [PATCH 305/507] In case something is getting rounded up to 10000000 --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 7f91d69..2f21029 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -750,7 +750,7 @@ and M.start_date=l.start_Date WHERE m.ValType_Cd in ('N','T') and m.MODIFIER_CD='@' -and (m.nval_num is null or m.nval_num<10000000 -- exclude lengths that exceed the spec +and (m.nval_num is null or m.nval_num<=9999999 -- exclude lengths that exceed the spec ; execute immediate 'create index lab_result_cm_idx on lab_result_cm (PATID, ENCOUNTERID)'; From 8cd2a055f575df45507528a4b2e530e68167efec Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 11 Jul 2017 09:08:54 -0500 Subject: [PATCH 306/507] Added missing parathesis --- Oracle/PCORNetLoader_ora.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 2f21029..8efb6ff 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -750,7 +750,7 @@ and M.start_date=l.start_Date WHERE m.ValType_Cd in ('N','T') and m.MODIFIER_CD='@' -and (m.nval_num is null or m.nval_num<=9999999 -- exclude lengths that exceed the spec +and (m.nval_num is null or m.nval_num<=9999999) -- exclude lengths that exceed the spec ; execute immediate 'create index lab_result_cm_idx on lab_result_cm (PATID, ENCOUNTERID)'; From c2cbde0aad0d782cf2883f097d4e6952ec1f2384 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 11 Jul 2017 10:28:42 -0500 Subject: [PATCH 307/507] Removed RESULT_NUM field type mapping which was causing an issue with CDM v3.1 --- SAS/data_step_view_prep.sas | 4 ---- 1 file changed, 4 deletions(-) diff --git a/SAS/data_step_view_prep.sas b/SAS/data_step_view_prep.sas index 0187957..d055c9f 100644 --- a/SAS/data_step_view_prep.sas +++ b/SAS/data_step_view_prep.sas @@ -148,7 +148,6 @@ data sasdata.LAB_RESULT_CM / view=sasdata.LAB_RESULT_CM; rename = ( RESULT_TIME = _RESULT_TIME SPECIMEN_TIME = _SPECIMEN_TIME - RESULT_NUM = _RESULT_NUM ) ) ; @@ -169,9 +168,6 @@ data sasdata.LAB_RESULT_CM / view=sasdata.LAB_RESULT_CM; SPECIMEN_TIME = input(_SPECIMEN_TIME, hhmmss.); format SPECIMEN_TIME hhmm.; drop _SPECIMEN_TIME; - - RESULT_NUM = put(_RESULT_NUM, best8.); - drop _RESULT_NUM; run; From 0616f5fc312e942c4bf387b8f3d280b509595c44 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Wed, 12 Jul 2017 11:20:33 -0500 Subject: [PATCH 308/507] Replace SCILHS lab normal ranges with KUMC normal ranges --- Oracle/PCORNetLoader_ora.sql | 23 +++- Oracle/labnormal.csv | 237 +++++++++++++++++++++++++++++++++++ Oracle/pmn_labnormal.csv | 12 -- Oracle/run-i2p-transform.sh | 2 +- 4 files changed, 255 insertions(+), 19 deletions(-) create mode 100644 Oracle/labnormal.csv delete mode 100644 Oracle/pmn_labnormal.csv diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 2a436bc..f85103c 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -711,10 +711,16 @@ CASE when length(m.Units_CD) > 11 then substr(m.Units_CD, 1, 11) ELSE TRIM(REPLACE(UPPER(m.Units_CD), '(CALC)', '')) end RESULT_UNIT, -- Local fix for KUMC -nullif(norm.NORM_RANGE_LOW,'') NORM_RANGE_LOW -,norm.NORM_MODIFIER_LOW, -nullif(norm.NORM_RANGE_HIGH,'') NORM_RANGE_HIGH -,norm.NORM_MODIFIER_HIGH, +norm.ref_lo NORM_RANGE_LOW, +case + when norm.ref_lo is not null then 'GE' + else null +end NORM_MODIFIER_LOW, +norm.ref_hi NORM_RANGE_HIGH, +case + when norm.ref_hi is not null then 'LE' + else null +end NORM_MODIFIER_HIGH, CASE NVL(nullif(m.VALUEFLAG_CD,''),'NI') WHEN 'H' THEN 'AH' WHEN 'L' THEN 'AL' WHEN 'A' THEN 'AB' ELSE 'NI' END ABN_IND, NULL RAW_LAB_NAME, NULL RAW_LAB_CODE, @@ -727,10 +733,9 @@ m.concept_cd RAW_FACILITY_CODE FROM i2b2fact M inner join encounter enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num -- Constraint to selected encounters - +inner join demographic demo on demo.patid=m.patient_num inner join pcornet_lab lab on lab.c_basecode = M.concept_cd and lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' inner JOIN pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname -left outer join pmn_labnormal norm on ont_parent.c_basecode=norm.LAB_NAME LEFT OUTER JOIN priority p @@ -747,6 +752,12 @@ and M.encounter_num=l.encounter_num and M.provider_id=l.provider_id and M.concept_cd=l.concept_Cd and M.start_date=l.start_Date + +LEFT OUTER JOIN labnormal norm + on m.concept_cd=norm.concept_cd + and demo.sex=norm.sex + and (m.start_date - demo.birth_date) > norm.age_lower + and (m.start_date - demo.birth_date) <= norm.age_upper WHERE m.ValType_Cd in ('N','T') and m.MODIFIER_CD='@'; diff --git a/Oracle/labnormal.csv b/Oracle/labnormal.csv new file mode 100644 index 0000000..5ce31b2 --- /dev/null +++ b/Oracle/labnormal.csv @@ -0,0 +1,237 @@ +"CONCEPT_CD","LAB_NAME","SEX","AGE_LOWER","AGE_UPPER","REF_LO","REF_HI" +"KUH|COMPONENT_ID:2070","LDH TOTAL","F",0,30,600,1395 +"KUH|COMPONENT_ID:2070","LDH TOTAL","F",30,512,600,1350 +"KUH|COMPONENT_ID:2070","LDH TOTAL","F",512,2816,495,1290 +"KUH|COMPONENT_ID:2070","LDH TOTAL","F",2816,4352,405,930 +"KUH|COMPONENT_ID:2070","LDH TOTAL","F",4352,25600,300,600 +"KUH|COMPONENT_ID:2070","LDH TOTAL","M",0,30,600,1395 +"KUH|COMPONENT_ID:2070","LDH TOTAL","M",30,512,600,1350 +"KUH|COMPONENT_ID:2070","LDH TOTAL","M",512,2816,495,1290 +"KUH|COMPONENT_ID:2070","LDH TOTAL","M",2816,4352,405,930 +"KUH|COMPONENT_ID:2070","LDH TOTAL","M",4352,25600,300,600 +"KUH|COMPONENT_ID:2321","LDL","F",0,25600,,100 +"KUH|COMPONENT_ID:2321","LDL","M",0,25600,,100 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","F",0,30,24,48 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","F",30,1536,35,71 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","F",1536,25600,24,44 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","M",0,30,24,48 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","M",30,1536,35,71 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","M",1536,25600,24,44 +"KUH|COMPONENT_ID:2020","MAGNESIUM","F",0,25600,1.6,2.6 +"KUH|COMPONENT_ID:2020","MAGNESIUM","M",0,25600,1.6,2.6 +"KUH|COMPONENT_ID:3043","MCH","F",0,25600,26,34 +"KUH|COMPONENT_ID:3043","MCH","M",0,25600,26,34 +"KUH|COMPONENT_ID:3044","MCHC","F",0,25600,32,36 +"KUH|COMPONENT_ID:3044","MCHC","M",0,25600,32,36 +"KUH|COMPONENT_ID:3042","MCV","F",0,25600,80,100 +"KUH|COMPONENT_ID:3042","MCV","M",0,25600,80,100 +"KUH|COMPONENT_ID:3030","MONOCYTES","F",0,25600,4,12 +"KUH|COMPONENT_ID:3030","MONOCYTES","M",0,25600,4,12 +"KUH|COMPONENT_ID:3045","MPV (PLATELET)","F",0,25600,7,11 +"KUH|COMPONENT_ID:3045","MPV (PLATELET)","M",0,25600,7,11 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","F",0,30,43,80 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","F",30,1536,28,56 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","F",1536,25600,41,77 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","M",0,30,43,80 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","M",30,1536,28,56 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","M",1536,25600,41,77 +"KUH|COMPONENT_ID:3026","NEUTROPHILS/100 LEUKOCYTES, SEG","F",0,25600,0,25 +"KUH|COMPONENT_ID:3026","NEUTROPHILS/100 LEUKOCYTES, SEG","M",0,25600,0,25 +"KUH|COMPONENT_ID:4003","PCO2, ART POC","F",0,14,27,40 +"KUH|COMPONENT_ID:4003","PCO2, ART POC","F",14,25600,35,45 +"KUH|COMPONENT_ID:4003","PCO2, ART POC","M",0,14,27,40 +"KUH|COMPONENT_ID:4003","PCO2, ART POC","M",14,25600,35,45 +"KUH|COMPONENT_ID:4001","PH, ART POC","F",0,1,7.29,7.45 +"KUH|COMPONENT_ID:4001","PH, ART POC","F",1,25600,7.35,7.45 +"KUH|COMPONENT_ID:4001","PH, ART POC","M",0,1,7.29,7.45 +"KUH|COMPONENT_ID:4001","PH, ART POC","M",1,25600,7.35,7.45 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","F",0,30,4,9 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","F",30,256,4,6 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","F",256,4608,3,5 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","F",4608,25600,2,4 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","M",0,30,4,9 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","M",30,256,4,6 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","M",256,4608,3,5 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","M",4608,25600,2,4 +"KUH|COMPONENT_ID:3053","PLATELET COUNT","F",0,25600,150,400 +"KUH|COMPONENT_ID:3008","PLATELET COUNT","F",0,25600,150,400 +"KUH|COMPONENT_ID:3053","PLATELET COUNT","M",0,25600,150,400 +"KUH|COMPONENT_ID:3008","PLATELET COUNT","M",0,25600,150,400 +"KUH|COMPONENT_ID:4005","PO2, ART POC","F",0,1,54,90 +"KUH|COMPONENT_ID:4005","PO2, ART POC","F",1,25600,80,100 +"KUH|COMPONENT_ID:4005","PO2, ART POC","M",0,1,54,90 +"KUH|COMPONENT_ID:4005","PO2, ART POC","M",1,25600,80,100 +"KUH|COMPONENT_ID:2002","POTASSIUM","F",0,25600,3.5,5.1 +"KUH|COMPONENT_ID:2002","POTASSIUM","M",0,25600,3.5,5.1 +"KUH|COMPONENT_ID:3085","PTT","F",0,25600,24,40 +"KUH|COMPONENT_ID:3085","PTT","M",0,25600,24,40 +"KUH|COMPONENT_ID:3041","RBC COUNT","F",0,30,4.7,5.9 +"KUH|COMPONENT_ID:3041","RBC COUNT","F",30,3584,3.3,5.2 +"KUH|COMPONENT_ID:3041","RBC COUNT","F",3584,25600,4,5 +"KUH|COMPONENT_ID:3041","RBC COUNT","M",0,30,4.7,5.9 +"KUH|COMPONENT_ID:3041","RBC COUNT","M",30,3584,3.3,5.2 +"KUH|COMPONENT_ID:3041","RBC COUNT","M",3584,25600,4.4,5.5 +"KUH|COMPONENT_ID:3046","RDW","F",0,25600,11,15 +"KUH|COMPONENT_ID:3046","RDW","M",0,25600,11,15 +"KUH|COMPONENT_ID:2000","SODIUM","F",0,14,134,144 +"KUH|COMPONENT_ID:2000","SODIUM","F",14,256,137,147 +"KUH|COMPONENT_ID:2000","SODIUM","F",256,25600,137,147 +"KUH|COMPONENT_ID:2000","SODIUM","M",0,14,134,144 +"KUH|COMPONENT_ID:2000","SODIUM","M",14,256,137,147 +"KUH|COMPONENT_ID:2000","SODIUM","M",256,25600,137,147 +"KUH|COMPONENT_ID:7050","SQUAMOUS EPI CELLS","F",0,25600,0,5 +"KUH|COMPONENT_ID:7050","SQUAMOUS EPI CELLS","M",0,25600,0,5 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","F",0,30,4.6,7 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","F",30,256,4.6,7.5 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","F",256,25600,6,8 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","M",0,30,4.6,7 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","M",30,256,4.6,7.5 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","M",256,25600,6,8 +"KUH|COMPONENT_ID:2319","TRIGLYCERIDE","F",0,25600,,150 +"KUH|COMPONENT_ID:2319","TRIGLYCERIDE","M",0,25600,,150 +"KUH|COMPONENT_ID:2326","TROPONIN I","F",0,25600,0,0.05 +"KUH|COMPONENT_ID:2326","TROPONIN I","M",0,25600,0,0.05 +"KUH|COMPONENT_ID:2198","TSH","F",0,25600,0.35,5 +"KUH|COMPONENT_ID:2198","TSH","M",0,25600,0.35,5 +"KUH|COMPONENT_ID:7006","URINE MACROSCOPIC, GLUCOSE","F",0,25600,, +"KUH|COMPONENT_ID:7006","URINE MACROSCOPIC, GLUCOSE","M",0,25600,, +"KUH|COMPONENT_ID:7015","URINE MACROSCOPIC, LEUKOCYTES","F",0,25600,, +"KUH|COMPONENT_ID:7015","URINE MACROSCOPIC, LEUKOCYTES","M",0,25600,, +"KUH|COMPONENT_ID:7014","URINE MACROSCOPIC, NITRITE","F",0,25600,, +"KUH|COMPONENT_ID:7014","URINE MACROSCOPIC, NITRITE","M",0,25600,, +"KUH|COMPONENT_ID:7004","URINE MACROSCOPIC, PROTEIN","F",0,25600,, +"KUH|COMPONENT_ID:7004","URINE MACROSCOPIC, PROTEIN","M",0,25600,, +"KUH|COMPONENT_ID:7009","URINE MACROSCOPIC, URINE BILE","F",0,25600,, +"KUH|COMPONENT_ID:7009","URINE MACROSCOPIC, URINE BILE","M",0,25600,, +"KUH|COMPONENT_ID:7011","URINE MACROSCOPIC, URINE BLOOD","F",0,25600,, +"KUH|COMPONENT_ID:7011","URINE MACROSCOPIC, URINE BLOOD","M",0,25600,, +"KUH|COMPONENT_ID:7008","URINE MACROSCOPIC, URINE KETONE","F",0,25600,, +"KUH|COMPONENT_ID:7008","URINE MACROSCOPIC, URINE KETONE","M",0,25600,, +"KUH|COMPONENT_ID:7003","URINE MACROSCOPIC, URINE PH","F",0,25600,5,8 +"KUH|COMPONENT_ID:7003","URINE MACROSCOPIC, URINE PH","M",0,25600,5,8 +"KUH|COMPONENT_ID:7002","URINE MACROSCOPIC, URINE SPEC GRAVITY","F",0,25600,1,1.04 +"KUH|COMPONENT_ID:7002","URINE MACROSCOPIC, URINE SPEC GRAVITY","M",0,25600,1,1.04 +"KUH|COMPONENT_ID:7012","URINE MACROSCOPIC, UROBILINOGEN","F",0,25600,, +"KUH|COMPONENT_ID:7012","URINE MACROSCOPIC, UROBILINOGEN","M",0,25600,, +"KUH|COMPONENT_ID:7017","URINE MICROSCOPIC, RBC","F",0,25600,0,3 +"KUH|COMPONENT_ID:7017","URINE MICROSCOPIC, RBC","M",0,25600,0,3 +"KUH|COMPONENT_ID:3009","WBC COUNT","F",0,30,5,20 +"KUH|COMPONENT_ID:3009","WBC COUNT","F",30,512,5,17 +"KUH|COMPONENT_ID:3009","WBC COUNT","F",512,4096,4.5,13 +"KUH|COMPONENT_ID:3009","WBC COUNT","F",4096,25600,4.5,11 +"KUH|COMPONENT_ID:3009","WBC COUNT","M",0,30,5,20 +"KUH|COMPONENT_ID:3009","WBC COUNT","M",30,512,5,17 +"KUH|COMPONENT_ID:3009","WBC COUNT","M",512,4096,4.5,13 +"KUH|COMPONENT_ID:3009","WBC COUNT","M",4096,25600,4.5,11 +"KUH|COMPONENT_ID:7016","WBC, URINE","F",0,25600,0,2 +"KUH|COMPONENT_ID:7016","WBC, URINE","M",0,25600,0,2 +"KUH|COMPONENT_ID:3027","WHITE BLOOD COUNT DIFFERENTIAL, BAND NEUTROPHIL","F",0,25600,0,10 +"KUH|COMPONENT_ID:3027","WHITE BLOOD COUNT DIFFERENTIAL, BAND NEUTROPHIL","M",0,25600,0,10 +"KUH|COMPONENT_ID:3023","WHITE BLOOD COUNT DIFFERENTIAL, BASOPHILS","F",0,25600,0,2 +"KUH|COMPONENT_ID:3023","WHITE BLOOD COUNT DIFFERENTIAL, BASOPHILS","M",0,25600,0,2 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","F",0,30,24,48 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","F",30,1536,35,71 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","F",1536,25600,24,44 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","M",0,30,24,48 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","M",30,1536,35,71 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","M",1536,25600,24,44 +"KUH|COMPONENT_ID:3025","WHITE CELL DIFFERENTIAL, ABSOLUTE BASO COUNT","F",4352,25600,0,0.2 +"KUH|COMPONENT_ID:3025","WHITE CELL DIFFERENTIAL, ABSOLUTE BASO COUNT","M",4352,25600,0,0.2 +"KUH|COMPONENT_ID:3022","WHITE CELL DIFFERENTIAL, ABSOLUTE EOS COUNT","F",4352,25600,0,0.45 +"KUH|COMPONENT_ID:3022","WHITE CELL DIFFERENTIAL, ABSOLUTE EOS COUNT","M",4352,25600,0,0.45 +"KUH|COMPONENT_ID:3016","WHITE CELL DIFFERENTIAL, ABSOLUTE LYMPH COUNT","F",4352,25600,1,4.8 +"KUH|COMPONENT_ID:3016","WHITE CELL DIFFERENTIAL, ABSOLUTE LYMPH COUNT","M",4352,25600,1,4.8 +"KUH|COMPONENT_ID:3019","WHITE CELL DIFFERENTIAL, ABSOLUTE MONO COUNT","F",4352,25600,0,0.8 +"KUH|COMPONENT_ID:3019","WHITE CELL DIFFERENTIAL, ABSOLUTE MONO COUNT","M",4352,25600,0,0.8 +"KUH|COMPONENT_ID:3012","WHITE CELL DIFFERENTIAL, ABSOLUTE NEUTROPHIL","F",4352,25600,1.8,7 +"KUH|COMPONENT_ID:3012","WHITE CELL DIFFERENTIAL, ABSOLUTE NEUTROPHIL","M",4352,25600,1.8,7 +"KUH|COMPONENT_ID:2023","ALBUMIN","F",0,25600,3.5,5 +"KUH|COMPONENT_ID:2023","ALBUMIN","M",0,25600,3.5,5 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",0,30,, +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",30,512,95,327 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",512,2560,99,232 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",2560,25600,25,110 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",0,30,, +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",30,512,95,327 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",512,2560,99,232 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",2560,25600,25,110 +"KUH|COMPONENT_ID:2065","ALT","F",0,256,4,76 +"KUH|COMPONENT_ID:2065","ALT","F",256,25600,7,56 +"KUH|COMPONENT_ID:2065","ALT","M",0,256,4,76 +"KUH|COMPONENT_ID:2065","ALT","M",256,25600,7,56 +"KUH|COMPONENT_ID:2006","ANION GAP","F",0,25600,3,12 +"KUH|COMPONENT_ID:2006","ANION GAP","M",0,25600,3,12 +"KUH|COMPONENT_ID:2064","AST","F",0,256,20,100 +"KUH|COMPONENT_ID:2064","AST","F",256,25600,7,40 +"KUH|COMPONENT_ID:2064","AST","M",0,256,20,100 +"KUH|COMPONENT_ID:2064","AST","M",256,25600,7,40 +"KUH|COMPONENT_ID:4007","BICARB, ART(CAL)","F",0,25600,21,28 +"KUH|COMPONENT_ID:4007","BICARB, ART(CAL)","M",0,25600,21,28 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",0,1,,2 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",1,3,,8 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",3,5,,12 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",5,6,0.3,1.2 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",6,25600,0.3,1.2 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",0,1,,2 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",1,3,,8 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",3,5,,12 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",5,6,0.3,1.2 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",6,25600,0.3,1.2 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","F",0,256,4,12 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","F",256,4608,5,20 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","F",4608,25600,7,25 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","M",0,256,4,12 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","M",256,4608,5,20 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","M",4608,25600,7,25 +"KUH|COMPONENT_ID:2017","CALCIUM","F",0,25600,8.5,10.6 +"KUH|COMPONENT_ID:2017","CALCIUM","M",0,25600,8.5,10.6 +"KUH|COMPONENT_ID:2018","CALCIUM, IONIZED POC","F",0,25600,1,1.3 +"KUH|COMPONENT_ID:2018","CALCIUM, IONIZED POC","M",0,25600,1,1.3 +"KUH|COMPONENT_ID:2004","CHLORIDE","F",0,25600,98,110 +"KUH|COMPONENT_ID:2004","CHLORIDE","M",0,25600,98,110 +"KUH|COMPONENT_ID:2318","CHOLESTEROL","F",0,25600,,200 +"KUH|COMPONENT_ID:2318","CHOLESTEROL","M",0,25600,,200 +"KUH|COMPONENT_ID:2005","CO2","F",0,14,19,24 +"KUH|COMPONENT_ID:2005","CO2","F",14,4608,20,28 +"KUH|COMPONENT_ID:2005","CO2","F",4608,25600,21,30 +"KUH|COMPONENT_ID:2005","CO2","M",0,14,19,24 +"KUH|COMPONENT_ID:2005","CO2","M",14,4608,20,28 +"KUH|COMPONENT_ID:2005","CO2","M",4608,25600,21,30 +"KUH|COMPONENT_ID:2009","CREATININE","F",0,30,0.3,1 +"KUH|COMPONENT_ID:2009","CREATININE","F",30,256,0.2,0.4 +"KUH|COMPONENT_ID:2009","CREATININE","F",256,4608,0.3,1 +"KUH|COMPONENT_ID:2009","CREATININE","F",4608,25600,0.4,1 +"KUH|COMPONENT_ID:2009","CREATININE","M",0,30,0.3,1 +"KUH|COMPONENT_ID:2009","CREATININE","M",30,256,0.2,0.4 +"KUH|COMPONENT_ID:2009","CREATININE","M",256,4608,0.3,1 +"KUH|COMPONENT_ID:2009","CREATININE","M",4608,25600,0.4,1.24 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","F",0,1,45,80 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","F",1,30,50,80 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","F",30,25600,70,100 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","M",0,1,45,80 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","M",1,30,50,80 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","M",30,25600,70,100 +"KUH|COMPONENT_ID:2320","HDL","F",0,25600,40, +"KUH|COMPONENT_ID:2320","HDL","M",0,25600,40, +"KUH|COMPONENT_ID:3004","HEMATOCRIT","F",0,30,51,65 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","F",30,180,38,50 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","F",180,1024,30,42 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","F",1024,3584,35,46 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","F",3584,25600,36,45 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","M",0,30,51,65 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","M",30,180,38,50 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","M",180,1024,30,42 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","M",1024,3584,36,46 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","M",3584,25600,40,50 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",0,30,17.5,21.5 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",30,180,12.8,15.5 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",180,1024,10,14 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",1024,3584,11,14 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",3584,25600,12,15 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",0,30,17.5,21.5 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",30,180,12.8,15.5 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",180,1024,10,14 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",1024,3584,11,14 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",3584,25600,13.5,16.5 +"KUH|COMPONENT_ID:2034","HEMOGLOBIN A1c","F",0,25600,4,6 +"KUH|COMPONENT_ID:2034","HEMOGLOBIN A1c","M",0,25600,4,6 diff --git a/Oracle/pmn_labnormal.csv b/Oracle/pmn_labnormal.csv deleted file mode 100644 index fd92568..0000000 --- a/Oracle/pmn_labnormal.csv +++ /dev/null @@ -1,12 +0,0 @@ -LAB_NAME,NORM_RANGE_LOW,NORM_MODIFIER_LOW,NORM_RANGE_HIGH,NORM_MODIFIER_HIGH -LAB_NAME:LDL,0,GE,165,LE -LAB_NAME:A1C,,NI,,NI -LAB_NAME:CK,50,GE,236,LE -LAB_NAME:CK_MB,,NI,,NI -LAB_NAME:CK_MBI,,NI,,NI -LAB_NAME:CREATININE,0,GE,1.6,LE -LAB_NAME:HGB,12,GE,17.5,LE -LAB_NAME:INR,0.8,GE,1.3,LE -LAB_NAME:TROP_I,0,GE,0.49,LE -LAB_NAME:TROP_T_QL,,NI,,NI -LAB_NAME:TROP_T_QN,0,GE,0.09,LE diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index 448cd8f..85ab0d9 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -2,7 +2,7 @@ set -e python load_csv.py harvest_local harvest_local.csv harvest_local.ctl pcornet_cdm_user pcornet_cdm -python load_csv.py PMN_LabNormal pmn_labnormal.csv pmn_labnormal.ctl pcornet_cdm_user pcornet_cdm +python load_csv.py LabNormal labnormal.csv labnormal.ctl pcornet_cdm_user pcornet_cdm # Run some tests sqlplus /nolog < Date: Thu, 13 Jul 2017 09:55:34 -0500 Subject: [PATCH 309/507] Break out result of enc_type coalesce before merging to `visit_dimension` --- Oracle/heron_encounter_style.sql | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 31d5c0e..a06d2e4 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -141,10 +141,12 @@ group by encounter_ide_source order by 1 ; */ +whenever sqlerror continue; +drop table enc_type_merge; +whenever sqlerror exit; -merge into "&&i2b2_data_schema".visit_dimension vd -using ( - with +create table enc_type_merge as +with enc_type_codes as ( select pm.pcori_path , replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code @@ -181,7 +183,12 @@ using ( case when ip is not null and ed is not null then 'EI' else coalesce(ei, ip, ed, os, e_is, av, oa, ot, ni, un, 'UN') end as pcori_code - from obs_enc_type_pivot) et + from obs_enc_type_pivot; + +--select pcori_code, count(*) from enc_type_merge group by pcori_code order by pcori_code; + +merge into "&&i2b2_data_schema".visit_dimension vd +using enc_type_merge et on ( vd.encounter_num = et.encounter_num ) when matched then update set inout_cd = et.pcori_code; From 180b8a5d5207abb248e0661105238b619e73dc7a Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Thu, 13 Jul 2017 15:46:11 -0500 Subject: [PATCH 310/507] Corrected lab normal range modifier values to align with CDM v3.1 spec --- Oracle/PCORNetLoader_ora.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index b272de9..2acf4d0 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -713,12 +713,12 @@ CASE end RESULT_UNIT, -- Local fix for KUMC norm.ref_lo NORM_RANGE_LOW, case - when norm.ref_lo is not null then 'GE' + when norm.ref_lo is not null then 'EQ' else null end NORM_MODIFIER_LOW, norm.ref_hi NORM_RANGE_HIGH, case - when norm.ref_hi is not null then 'LE' + when norm.ref_hi is not null then 'EQ' else null end NORM_MODIFIER_HIGH, CASE NVL(nullif(m.VALUEFLAG_CD,''),'NI') WHEN 'H' THEN 'AH' WHEN 'L' THEN 'AL' WHEN 'A' THEN 'AB' ELSE 'NI' END ABN_IND, From c79c97c5806a758e5bc0f67f40337d74d38fe372 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 26 Sep 2017 11:00:03 -0500 Subject: [PATCH 311/507] Fixed normal range modifiers --- Oracle/PCORNetLoader_ora.sql | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 2acf4d0..a2ecd11 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1,4 +1,4 @@ -------------------------------------------------------------------------------------------- +--------------------------------------------------------------------------------------------- -- PCORNetLoader Script -- Orignal MSSQL Verion Contributors: Jeff Klann, PhD; Aaron Abend; Arturo Torres -- Translate to Oracle version: by Kun Wei(Wake Forest) @@ -713,13 +713,17 @@ CASE end RESULT_UNIT, -- Local fix for KUMC norm.ref_lo NORM_RANGE_LOW, case - when norm.ref_lo is not null then 'EQ' - else null + when norm.ref_lo is not null and norm.ref_hi is not null then 'EQ' + when norm.ref_lo is not null and norm.ref_hi is null then 'GE' + when norm.ref_lo is null and norm.ref_hi is not null then 'NO' +else 'NI' end NORM_MODIFIER_LOW, norm.ref_hi NORM_RANGE_HIGH, case - when norm.ref_hi is not null then 'EQ' - else null + when norm.ref_lo is not null and norm.ref_hi is not null then 'EQ' + when norm.ref_lo is not null and norm.ref_hi is null then 'NO' + when norm.ref_lo is null and norm.ref_hi is not null then 'LE' + else 'NI' end NORM_MODIFIER_HIGH, CASE NVL(nullif(m.VALUEFLAG_CD,''),'NI') WHEN 'H' THEN 'AH' WHEN 'L' THEN 'AL' WHEN 'A' THEN 'AB' ELSE 'NI' END ABN_IND, NULL RAW_LAB_NAME, From 2772a0d15b017204559d0230b630b4ad8b97b697 Mon Sep 17 00:00:00 2001 From: Michael Prittie Date: Tue, 26 Sep 2017 11:14:08 -0500 Subject: [PATCH 312/507] Fixed AGE_UPPER to 100 years in days --- Oracle/labnormal.csv | 268 +++++++++++++++++++++---------------------- 1 file changed, 134 insertions(+), 134 deletions(-) diff --git a/Oracle/labnormal.csv b/Oracle/labnormal.csv index 5ce31b2..b6f18ee 100644 --- a/Oracle/labnormal.csv +++ b/Oracle/labnormal.csv @@ -3,235 +3,235 @@ "KUH|COMPONENT_ID:2070","LDH TOTAL","F",30,512,600,1350 "KUH|COMPONENT_ID:2070","LDH TOTAL","F",512,2816,495,1290 "KUH|COMPONENT_ID:2070","LDH TOTAL","F",2816,4352,405,930 -"KUH|COMPONENT_ID:2070","LDH TOTAL","F",4352,25600,300,600 +"KUH|COMPONENT_ID:2070","LDH TOTAL","F",4352,36500,300,600 "KUH|COMPONENT_ID:2070","LDH TOTAL","M",0,30,600,1395 "KUH|COMPONENT_ID:2070","LDH TOTAL","M",30,512,600,1350 "KUH|COMPONENT_ID:2070","LDH TOTAL","M",512,2816,495,1290 "KUH|COMPONENT_ID:2070","LDH TOTAL","M",2816,4352,405,930 -"KUH|COMPONENT_ID:2070","LDH TOTAL","M",4352,25600,300,600 -"KUH|COMPONENT_ID:2321","LDL","F",0,25600,,100 -"KUH|COMPONENT_ID:2321","LDL","M",0,25600,,100 +"KUH|COMPONENT_ID:2070","LDH TOTAL","M",4352,36500,300,600 +"KUH|COMPONENT_ID:2321","LDL","F",0,36500,,100 +"KUH|COMPONENT_ID:2321","LDL","M",0,36500,,100 "KUH|COMPONENT_ID:3028","LYMPHOCYTES","F",0,30,24,48 "KUH|COMPONENT_ID:3028","LYMPHOCYTES","F",30,1536,35,71 -"KUH|COMPONENT_ID:3028","LYMPHOCYTES","F",1536,25600,24,44 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","F",1536,36500,24,44 "KUH|COMPONENT_ID:3028","LYMPHOCYTES","M",0,30,24,48 "KUH|COMPONENT_ID:3028","LYMPHOCYTES","M",30,1536,35,71 -"KUH|COMPONENT_ID:3028","LYMPHOCYTES","M",1536,25600,24,44 -"KUH|COMPONENT_ID:2020","MAGNESIUM","F",0,25600,1.6,2.6 -"KUH|COMPONENT_ID:2020","MAGNESIUM","M",0,25600,1.6,2.6 -"KUH|COMPONENT_ID:3043","MCH","F",0,25600,26,34 -"KUH|COMPONENT_ID:3043","MCH","M",0,25600,26,34 -"KUH|COMPONENT_ID:3044","MCHC","F",0,25600,32,36 -"KUH|COMPONENT_ID:3044","MCHC","M",0,25600,32,36 -"KUH|COMPONENT_ID:3042","MCV","F",0,25600,80,100 -"KUH|COMPONENT_ID:3042","MCV","M",0,25600,80,100 -"KUH|COMPONENT_ID:3030","MONOCYTES","F",0,25600,4,12 -"KUH|COMPONENT_ID:3030","MONOCYTES","M",0,25600,4,12 -"KUH|COMPONENT_ID:3045","MPV (PLATELET)","F",0,25600,7,11 -"KUH|COMPONENT_ID:3045","MPV (PLATELET)","M",0,25600,7,11 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","M",1536,36500,24,44 +"KUH|COMPONENT_ID:2020","MAGNESIUM","F",0,36500,1.6,2.6 +"KUH|COMPONENT_ID:2020","MAGNESIUM","M",0,36500,1.6,2.6 +"KUH|COMPONENT_ID:3043","MCH","F",0,36500,26,34 +"KUH|COMPONENT_ID:3043","MCH","M",0,36500,26,34 +"KUH|COMPONENT_ID:3044","MCHC","F",0,36500,32,36 +"KUH|COMPONENT_ID:3044","MCHC","M",0,36500,32,36 +"KUH|COMPONENT_ID:3042","MCV","F",0,36500,80,100 +"KUH|COMPONENT_ID:3042","MCV","M",0,36500,80,100 +"KUH|COMPONENT_ID:3030","MONOCYTES","F",0,36500,4,12 +"KUH|COMPONENT_ID:3030","MONOCYTES","M",0,36500,4,12 +"KUH|COMPONENT_ID:3045","MPV (PLATELET)","F",0,36500,7,11 +"KUH|COMPONENT_ID:3045","MPV (PLATELET)","M",0,36500,7,11 "KUH|COMPONENT_ID:3010","NEUTROPHILS","F",0,30,43,80 "KUH|COMPONENT_ID:3010","NEUTROPHILS","F",30,1536,28,56 -"KUH|COMPONENT_ID:3010","NEUTROPHILS","F",1536,25600,41,77 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","F",1536,36500,41,77 "KUH|COMPONENT_ID:3010","NEUTROPHILS","M",0,30,43,80 "KUH|COMPONENT_ID:3010","NEUTROPHILS","M",30,1536,28,56 -"KUH|COMPONENT_ID:3010","NEUTROPHILS","M",1536,25600,41,77 -"KUH|COMPONENT_ID:3026","NEUTROPHILS/100 LEUKOCYTES, SEG","F",0,25600,0,25 -"KUH|COMPONENT_ID:3026","NEUTROPHILS/100 LEUKOCYTES, SEG","M",0,25600,0,25 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","M",1536,36500,41,77 +"KUH|COMPONENT_ID:3026","NEUTROPHILS/100 LEUKOCYTES, SEG","F",0,36500,0,25 +"KUH|COMPONENT_ID:3026","NEUTROPHILS/100 LEUKOCYTES, SEG","M",0,36500,0,25 "KUH|COMPONENT_ID:4003","PCO2, ART POC","F",0,14,27,40 -"KUH|COMPONENT_ID:4003","PCO2, ART POC","F",14,25600,35,45 +"KUH|COMPONENT_ID:4003","PCO2, ART POC","F",14,36500,35,45 "KUH|COMPONENT_ID:4003","PCO2, ART POC","M",0,14,27,40 -"KUH|COMPONENT_ID:4003","PCO2, ART POC","M",14,25600,35,45 +"KUH|COMPONENT_ID:4003","PCO2, ART POC","M",14,36500,35,45 "KUH|COMPONENT_ID:4001","PH, ART POC","F",0,1,7.29,7.45 -"KUH|COMPONENT_ID:4001","PH, ART POC","F",1,25600,7.35,7.45 +"KUH|COMPONENT_ID:4001","PH, ART POC","F",1,36500,7.35,7.45 "KUH|COMPONENT_ID:4001","PH, ART POC","M",0,1,7.29,7.45 -"KUH|COMPONENT_ID:4001","PH, ART POC","M",1,25600,7.35,7.45 +"KUH|COMPONENT_ID:4001","PH, ART POC","M",1,36500,7.35,7.45 "KUH|COMPONENT_ID:2033","PHOSPHORUS","F",0,30,4,9 "KUH|COMPONENT_ID:2033","PHOSPHORUS","F",30,256,4,6 "KUH|COMPONENT_ID:2033","PHOSPHORUS","F",256,4608,3,5 -"KUH|COMPONENT_ID:2033","PHOSPHORUS","F",4608,25600,2,4 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","F",4608,36500,2,4 "KUH|COMPONENT_ID:2033","PHOSPHORUS","M",0,30,4,9 "KUH|COMPONENT_ID:2033","PHOSPHORUS","M",30,256,4,6 "KUH|COMPONENT_ID:2033","PHOSPHORUS","M",256,4608,3,5 -"KUH|COMPONENT_ID:2033","PHOSPHORUS","M",4608,25600,2,4 -"KUH|COMPONENT_ID:3053","PLATELET COUNT","F",0,25600,150,400 -"KUH|COMPONENT_ID:3008","PLATELET COUNT","F",0,25600,150,400 -"KUH|COMPONENT_ID:3053","PLATELET COUNT","M",0,25600,150,400 -"KUH|COMPONENT_ID:3008","PLATELET COUNT","M",0,25600,150,400 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","M",4608,36500,2,4 +"KUH|COMPONENT_ID:3053","PLATELET COUNT","F",0,36500,150,400 +"KUH|COMPONENT_ID:3008","PLATELET COUNT","F",0,36500,150,400 +"KUH|COMPONENT_ID:3053","PLATELET COUNT","M",0,36500,150,400 +"KUH|COMPONENT_ID:3008","PLATELET COUNT","M",0,36500,150,400 "KUH|COMPONENT_ID:4005","PO2, ART POC","F",0,1,54,90 -"KUH|COMPONENT_ID:4005","PO2, ART POC","F",1,25600,80,100 +"KUH|COMPONENT_ID:4005","PO2, ART POC","F",1,36500,80,100 "KUH|COMPONENT_ID:4005","PO2, ART POC","M",0,1,54,90 -"KUH|COMPONENT_ID:4005","PO2, ART POC","M",1,25600,80,100 -"KUH|COMPONENT_ID:2002","POTASSIUM","F",0,25600,3.5,5.1 -"KUH|COMPONENT_ID:2002","POTASSIUM","M",0,25600,3.5,5.1 -"KUH|COMPONENT_ID:3085","PTT","F",0,25600,24,40 -"KUH|COMPONENT_ID:3085","PTT","M",0,25600,24,40 +"KUH|COMPONENT_ID:4005","PO2, ART POC","M",1,36500,80,100 +"KUH|COMPONENT_ID:2002","POTASSIUM","F",0,36500,3.5,5.1 +"KUH|COMPONENT_ID:2002","POTASSIUM","M",0,36500,3.5,5.1 +"KUH|COMPONENT_ID:3085","PTT","F",0,36500,24,40 +"KUH|COMPONENT_ID:3085","PTT","M",0,36500,24,40 "KUH|COMPONENT_ID:3041","RBC COUNT","F",0,30,4.7,5.9 "KUH|COMPONENT_ID:3041","RBC COUNT","F",30,3584,3.3,5.2 -"KUH|COMPONENT_ID:3041","RBC COUNT","F",3584,25600,4,5 +"KUH|COMPONENT_ID:3041","RBC COUNT","F",3584,36500,4,5 "KUH|COMPONENT_ID:3041","RBC COUNT","M",0,30,4.7,5.9 "KUH|COMPONENT_ID:3041","RBC COUNT","M",30,3584,3.3,5.2 -"KUH|COMPONENT_ID:3041","RBC COUNT","M",3584,25600,4.4,5.5 -"KUH|COMPONENT_ID:3046","RDW","F",0,25600,11,15 -"KUH|COMPONENT_ID:3046","RDW","M",0,25600,11,15 +"KUH|COMPONENT_ID:3041","RBC COUNT","M",3584,36500,4.4,5.5 +"KUH|COMPONENT_ID:3046","RDW","F",0,36500,11,15 +"KUH|COMPONENT_ID:3046","RDW","M",0,36500,11,15 "KUH|COMPONENT_ID:2000","SODIUM","F",0,14,134,144 "KUH|COMPONENT_ID:2000","SODIUM","F",14,256,137,147 -"KUH|COMPONENT_ID:2000","SODIUM","F",256,25600,137,147 +"KUH|COMPONENT_ID:2000","SODIUM","F",256,36500,137,147 "KUH|COMPONENT_ID:2000","SODIUM","M",0,14,134,144 "KUH|COMPONENT_ID:2000","SODIUM","M",14,256,137,147 -"KUH|COMPONENT_ID:2000","SODIUM","M",256,25600,137,147 -"KUH|COMPONENT_ID:7050","SQUAMOUS EPI CELLS","F",0,25600,0,5 -"KUH|COMPONENT_ID:7050","SQUAMOUS EPI CELLS","M",0,25600,0,5 +"KUH|COMPONENT_ID:2000","SODIUM","M",256,36500,137,147 +"KUH|COMPONENT_ID:7050","SQUAMOUS EPI CELLS","F",0,36500,0,5 +"KUH|COMPONENT_ID:7050","SQUAMOUS EPI CELLS","M",0,36500,0,5 "KUH|COMPONENT_ID:2032","TOTAL PROTEIN","F",0,30,4.6,7 "KUH|COMPONENT_ID:2032","TOTAL PROTEIN","F",30,256,4.6,7.5 -"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","F",256,25600,6,8 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","F",256,36500,6,8 "KUH|COMPONENT_ID:2032","TOTAL PROTEIN","M",0,30,4.6,7 "KUH|COMPONENT_ID:2032","TOTAL PROTEIN","M",30,256,4.6,7.5 -"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","M",256,25600,6,8 -"KUH|COMPONENT_ID:2319","TRIGLYCERIDE","F",0,25600,,150 -"KUH|COMPONENT_ID:2319","TRIGLYCERIDE","M",0,25600,,150 -"KUH|COMPONENT_ID:2326","TROPONIN I","F",0,25600,0,0.05 -"KUH|COMPONENT_ID:2326","TROPONIN I","M",0,25600,0,0.05 -"KUH|COMPONENT_ID:2198","TSH","F",0,25600,0.35,5 -"KUH|COMPONENT_ID:2198","TSH","M",0,25600,0.35,5 -"KUH|COMPONENT_ID:7006","URINE MACROSCOPIC, GLUCOSE","F",0,25600,, -"KUH|COMPONENT_ID:7006","URINE MACROSCOPIC, GLUCOSE","M",0,25600,, -"KUH|COMPONENT_ID:7015","URINE MACROSCOPIC, LEUKOCYTES","F",0,25600,, -"KUH|COMPONENT_ID:7015","URINE MACROSCOPIC, LEUKOCYTES","M",0,25600,, -"KUH|COMPONENT_ID:7014","URINE MACROSCOPIC, NITRITE","F",0,25600,, -"KUH|COMPONENT_ID:7014","URINE MACROSCOPIC, NITRITE","M",0,25600,, -"KUH|COMPONENT_ID:7004","URINE MACROSCOPIC, PROTEIN","F",0,25600,, -"KUH|COMPONENT_ID:7004","URINE MACROSCOPIC, PROTEIN","M",0,25600,, -"KUH|COMPONENT_ID:7009","URINE MACROSCOPIC, URINE BILE","F",0,25600,, -"KUH|COMPONENT_ID:7009","URINE MACROSCOPIC, URINE BILE","M",0,25600,, -"KUH|COMPONENT_ID:7011","URINE MACROSCOPIC, URINE BLOOD","F",0,25600,, -"KUH|COMPONENT_ID:7011","URINE MACROSCOPIC, URINE BLOOD","M",0,25600,, -"KUH|COMPONENT_ID:7008","URINE MACROSCOPIC, URINE KETONE","F",0,25600,, -"KUH|COMPONENT_ID:7008","URINE MACROSCOPIC, URINE KETONE","M",0,25600,, -"KUH|COMPONENT_ID:7003","URINE MACROSCOPIC, URINE PH","F",0,25600,5,8 -"KUH|COMPONENT_ID:7003","URINE MACROSCOPIC, URINE PH","M",0,25600,5,8 -"KUH|COMPONENT_ID:7002","URINE MACROSCOPIC, URINE SPEC GRAVITY","F",0,25600,1,1.04 -"KUH|COMPONENT_ID:7002","URINE MACROSCOPIC, URINE SPEC GRAVITY","M",0,25600,1,1.04 -"KUH|COMPONENT_ID:7012","URINE MACROSCOPIC, UROBILINOGEN","F",0,25600,, -"KUH|COMPONENT_ID:7012","URINE MACROSCOPIC, UROBILINOGEN","M",0,25600,, -"KUH|COMPONENT_ID:7017","URINE MICROSCOPIC, RBC","F",0,25600,0,3 -"KUH|COMPONENT_ID:7017","URINE MICROSCOPIC, RBC","M",0,25600,0,3 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","M",256,36500,6,8 +"KUH|COMPONENT_ID:2319","TRIGLYCERIDE","F",0,36500,,150 +"KUH|COMPONENT_ID:2319","TRIGLYCERIDE","M",0,36500,,150 +"KUH|COMPONENT_ID:2326","TROPONIN I","F",0,36500,0,0.05 +"KUH|COMPONENT_ID:2326","TROPONIN I","M",0,36500,0,0.05 +"KUH|COMPONENT_ID:2198","TSH","F",0,36500,0.35,5 +"KUH|COMPONENT_ID:2198","TSH","M",0,36500,0.35,5 +"KUH|COMPONENT_ID:7006","URINE MACROSCOPIC, GLUCOSE","F",0,36500,, +"KUH|COMPONENT_ID:7006","URINE MACROSCOPIC, GLUCOSE","M",0,36500,, +"KUH|COMPONENT_ID:7015","URINE MACROSCOPIC, LEUKOCYTES","F",0,36500,, +"KUH|COMPONENT_ID:7015","URINE MACROSCOPIC, LEUKOCYTES","M",0,36500,, +"KUH|COMPONENT_ID:7014","URINE MACROSCOPIC, NITRITE","F",0,36500,, +"KUH|COMPONENT_ID:7014","URINE MACROSCOPIC, NITRITE","M",0,36500,, +"KUH|COMPONENT_ID:7004","URINE MACROSCOPIC, PROTEIN","F",0,36500,, +"KUH|COMPONENT_ID:7004","URINE MACROSCOPIC, PROTEIN","M",0,36500,, +"KUH|COMPONENT_ID:7009","URINE MACROSCOPIC, URINE BILE","F",0,36500,, +"KUH|COMPONENT_ID:7009","URINE MACROSCOPIC, URINE BILE","M",0,36500,, +"KUH|COMPONENT_ID:7011","URINE MACROSCOPIC, URINE BLOOD","F",0,36500,, +"KUH|COMPONENT_ID:7011","URINE MACROSCOPIC, URINE BLOOD","M",0,36500,, +"KUH|COMPONENT_ID:7008","URINE MACROSCOPIC, URINE KETONE","F",0,36500,, +"KUH|COMPONENT_ID:7008","URINE MACROSCOPIC, URINE KETONE","M",0,36500,, +"KUH|COMPONENT_ID:7003","URINE MACROSCOPIC, URINE PH","F",0,36500,5,8 +"KUH|COMPONENT_ID:7003","URINE MACROSCOPIC, URINE PH","M",0,36500,5,8 +"KUH|COMPONENT_ID:7002","URINE MACROSCOPIC, URINE SPEC GRAVITY","F",0,36500,1,1.04 +"KUH|COMPONENT_ID:7002","URINE MACROSCOPIC, URINE SPEC GRAVITY","M",0,36500,1,1.04 +"KUH|COMPONENT_ID:7012","URINE MACROSCOPIC, UROBILINOGEN","F",0,36500,, +"KUH|COMPONENT_ID:7012","URINE MACROSCOPIC, UROBILINOGEN","M",0,36500,, +"KUH|COMPONENT_ID:7017","URINE MICROSCOPIC, RBC","F",0,36500,0,3 +"KUH|COMPONENT_ID:7017","URINE MICROSCOPIC, RBC","M",0,36500,0,3 "KUH|COMPONENT_ID:3009","WBC COUNT","F",0,30,5,20 "KUH|COMPONENT_ID:3009","WBC COUNT","F",30,512,5,17 "KUH|COMPONENT_ID:3009","WBC COUNT","F",512,4096,4.5,13 -"KUH|COMPONENT_ID:3009","WBC COUNT","F",4096,25600,4.5,11 +"KUH|COMPONENT_ID:3009","WBC COUNT","F",4096,36500,4.5,11 "KUH|COMPONENT_ID:3009","WBC COUNT","M",0,30,5,20 "KUH|COMPONENT_ID:3009","WBC COUNT","M",30,512,5,17 "KUH|COMPONENT_ID:3009","WBC COUNT","M",512,4096,4.5,13 -"KUH|COMPONENT_ID:3009","WBC COUNT","M",4096,25600,4.5,11 -"KUH|COMPONENT_ID:7016","WBC, URINE","F",0,25600,0,2 -"KUH|COMPONENT_ID:7016","WBC, URINE","M",0,25600,0,2 -"KUH|COMPONENT_ID:3027","WHITE BLOOD COUNT DIFFERENTIAL, BAND NEUTROPHIL","F",0,25600,0,10 -"KUH|COMPONENT_ID:3027","WHITE BLOOD COUNT DIFFERENTIAL, BAND NEUTROPHIL","M",0,25600,0,10 -"KUH|COMPONENT_ID:3023","WHITE BLOOD COUNT DIFFERENTIAL, BASOPHILS","F",0,25600,0,2 -"KUH|COMPONENT_ID:3023","WHITE BLOOD COUNT DIFFERENTIAL, BASOPHILS","M",0,25600,0,2 +"KUH|COMPONENT_ID:3009","WBC COUNT","M",4096,36500,4.5,11 +"KUH|COMPONENT_ID:7016","WBC, URINE","F",0,36500,0,2 +"KUH|COMPONENT_ID:7016","WBC, URINE","M",0,36500,0,2 +"KUH|COMPONENT_ID:3027","WHITE BLOOD COUNT DIFFERENTIAL, BAND NEUTROPHIL","F",0,36500,0,10 +"KUH|COMPONENT_ID:3027","WHITE BLOOD COUNT DIFFERENTIAL, BAND NEUTROPHIL","M",0,36500,0,10 +"KUH|COMPONENT_ID:3023","WHITE BLOOD COUNT DIFFERENTIAL, BASOPHILS","F",0,36500,0,2 +"KUH|COMPONENT_ID:3023","WHITE BLOOD COUNT DIFFERENTIAL, BASOPHILS","M",0,36500,0,2 "KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","F",0,30,24,48 "KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","F",30,1536,35,71 -"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","F",1536,25600,24,44 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","F",1536,36500,24,44 "KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","M",0,30,24,48 "KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","M",30,1536,35,71 -"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","M",1536,25600,24,44 -"KUH|COMPONENT_ID:3025","WHITE CELL DIFFERENTIAL, ABSOLUTE BASO COUNT","F",4352,25600,0,0.2 -"KUH|COMPONENT_ID:3025","WHITE CELL DIFFERENTIAL, ABSOLUTE BASO COUNT","M",4352,25600,0,0.2 -"KUH|COMPONENT_ID:3022","WHITE CELL DIFFERENTIAL, ABSOLUTE EOS COUNT","F",4352,25600,0,0.45 -"KUH|COMPONENT_ID:3022","WHITE CELL DIFFERENTIAL, ABSOLUTE EOS COUNT","M",4352,25600,0,0.45 -"KUH|COMPONENT_ID:3016","WHITE CELL DIFFERENTIAL, ABSOLUTE LYMPH COUNT","F",4352,25600,1,4.8 -"KUH|COMPONENT_ID:3016","WHITE CELL DIFFERENTIAL, ABSOLUTE LYMPH COUNT","M",4352,25600,1,4.8 -"KUH|COMPONENT_ID:3019","WHITE CELL DIFFERENTIAL, ABSOLUTE MONO COUNT","F",4352,25600,0,0.8 -"KUH|COMPONENT_ID:3019","WHITE CELL DIFFERENTIAL, ABSOLUTE MONO COUNT","M",4352,25600,0,0.8 -"KUH|COMPONENT_ID:3012","WHITE CELL DIFFERENTIAL, ABSOLUTE NEUTROPHIL","F",4352,25600,1.8,7 -"KUH|COMPONENT_ID:3012","WHITE CELL DIFFERENTIAL, ABSOLUTE NEUTROPHIL","M",4352,25600,1.8,7 -"KUH|COMPONENT_ID:2023","ALBUMIN","F",0,25600,3.5,5 -"KUH|COMPONENT_ID:2023","ALBUMIN","M",0,25600,3.5,5 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","M",1536,36500,24,44 +"KUH|COMPONENT_ID:3025","WHITE CELL DIFFERENTIAL, ABSOLUTE BASO COUNT","F",4352,36500,0,0.2 +"KUH|COMPONENT_ID:3025","WHITE CELL DIFFERENTIAL, ABSOLUTE BASO COUNT","M",4352,36500,0,0.2 +"KUH|COMPONENT_ID:3022","WHITE CELL DIFFERENTIAL, ABSOLUTE EOS COUNT","F",4352,36500,0,0.45 +"KUH|COMPONENT_ID:3022","WHITE CELL DIFFERENTIAL, ABSOLUTE EOS COUNT","M",4352,36500,0,0.45 +"KUH|COMPONENT_ID:3016","WHITE CELL DIFFERENTIAL, ABSOLUTE LYMPH COUNT","F",4352,36500,1,4.8 +"KUH|COMPONENT_ID:3016","WHITE CELL DIFFERENTIAL, ABSOLUTE LYMPH COUNT","M",4352,36500,1,4.8 +"KUH|COMPONENT_ID:3019","WHITE CELL DIFFERENTIAL, ABSOLUTE MONO COUNT","F",4352,36500,0,0.8 +"KUH|COMPONENT_ID:3019","WHITE CELL DIFFERENTIAL, ABSOLUTE MONO COUNT","M",4352,36500,0,0.8 +"KUH|COMPONENT_ID:3012","WHITE CELL DIFFERENTIAL, ABSOLUTE NEUTROPHIL","F",4352,36500,1.8,7 +"KUH|COMPONENT_ID:3012","WHITE CELL DIFFERENTIAL, ABSOLUTE NEUTROPHIL","M",4352,36500,1.8,7 +"KUH|COMPONENT_ID:2023","ALBUMIN","F",0,36500,3.5,5 +"KUH|COMPONENT_ID:2023","ALBUMIN","M",0,36500,3.5,5 "KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",0,30,, "KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",30,512,95,327 "KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",512,2560,99,232 -"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",2560,25600,25,110 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",2560,36500,25,110 "KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",0,30,, "KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",30,512,95,327 "KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",512,2560,99,232 -"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",2560,25600,25,110 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",2560,36500,25,110 "KUH|COMPONENT_ID:2065","ALT","F",0,256,4,76 -"KUH|COMPONENT_ID:2065","ALT","F",256,25600,7,56 +"KUH|COMPONENT_ID:2065","ALT","F",256,36500,7,56 "KUH|COMPONENT_ID:2065","ALT","M",0,256,4,76 -"KUH|COMPONENT_ID:2065","ALT","M",256,25600,7,56 -"KUH|COMPONENT_ID:2006","ANION GAP","F",0,25600,3,12 -"KUH|COMPONENT_ID:2006","ANION GAP","M",0,25600,3,12 +"KUH|COMPONENT_ID:2065","ALT","M",256,36500,7,56 +"KUH|COMPONENT_ID:2006","ANION GAP","F",0,36500,3,12 +"KUH|COMPONENT_ID:2006","ANION GAP","M",0,36500,3,12 "KUH|COMPONENT_ID:2064","AST","F",0,256,20,100 -"KUH|COMPONENT_ID:2064","AST","F",256,25600,7,40 +"KUH|COMPONENT_ID:2064","AST","F",256,36500,7,40 "KUH|COMPONENT_ID:2064","AST","M",0,256,20,100 -"KUH|COMPONENT_ID:2064","AST","M",256,25600,7,40 -"KUH|COMPONENT_ID:4007","BICARB, ART(CAL)","F",0,25600,21,28 -"KUH|COMPONENT_ID:4007","BICARB, ART(CAL)","M",0,25600,21,28 +"KUH|COMPONENT_ID:2064","AST","M",256,36500,7,40 +"KUH|COMPONENT_ID:4007","BICARB, ART(CAL)","F",0,36500,21,28 +"KUH|COMPONENT_ID:4007","BICARB, ART(CAL)","M",0,36500,21,28 "KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",0,1,,2 "KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",1,3,,8 "KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",3,5,,12 "KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",5,6,0.3,1.2 -"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",6,25600,0.3,1.2 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",6,36500,0.3,1.2 "KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",0,1,,2 "KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",1,3,,8 "KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",3,5,,12 "KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",5,6,0.3,1.2 -"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",6,25600,0.3,1.2 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",6,36500,0.3,1.2 "KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","F",0,256,4,12 "KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","F",256,4608,5,20 -"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","F",4608,25600,7,25 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","F",4608,36500,7,25 "KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","M",0,256,4,12 "KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","M",256,4608,5,20 -"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","M",4608,25600,7,25 -"KUH|COMPONENT_ID:2017","CALCIUM","F",0,25600,8.5,10.6 -"KUH|COMPONENT_ID:2017","CALCIUM","M",0,25600,8.5,10.6 -"KUH|COMPONENT_ID:2018","CALCIUM, IONIZED POC","F",0,25600,1,1.3 -"KUH|COMPONENT_ID:2018","CALCIUM, IONIZED POC","M",0,25600,1,1.3 -"KUH|COMPONENT_ID:2004","CHLORIDE","F",0,25600,98,110 -"KUH|COMPONENT_ID:2004","CHLORIDE","M",0,25600,98,110 -"KUH|COMPONENT_ID:2318","CHOLESTEROL","F",0,25600,,200 -"KUH|COMPONENT_ID:2318","CHOLESTEROL","M",0,25600,,200 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","M",4608,36500,7,25 +"KUH|COMPONENT_ID:2017","CALCIUM","F",0,36500,8.5,10.6 +"KUH|COMPONENT_ID:2017","CALCIUM","M",0,36500,8.5,10.6 +"KUH|COMPONENT_ID:2018","CALCIUM, IONIZED POC","F",0,36500,1,1.3 +"KUH|COMPONENT_ID:2018","CALCIUM, IONIZED POC","M",0,36500,1,1.3 +"KUH|COMPONENT_ID:2004","CHLORIDE","F",0,36500,98,110 +"KUH|COMPONENT_ID:2004","CHLORIDE","M",0,36500,98,110 +"KUH|COMPONENT_ID:2318","CHOLESTEROL","F",0,36500,,200 +"KUH|COMPONENT_ID:2318","CHOLESTEROL","M",0,36500,,200 "KUH|COMPONENT_ID:2005","CO2","F",0,14,19,24 "KUH|COMPONENT_ID:2005","CO2","F",14,4608,20,28 -"KUH|COMPONENT_ID:2005","CO2","F",4608,25600,21,30 +"KUH|COMPONENT_ID:2005","CO2","F",4608,36500,21,30 "KUH|COMPONENT_ID:2005","CO2","M",0,14,19,24 "KUH|COMPONENT_ID:2005","CO2","M",14,4608,20,28 -"KUH|COMPONENT_ID:2005","CO2","M",4608,25600,21,30 +"KUH|COMPONENT_ID:2005","CO2","M",4608,36500,21,30 "KUH|COMPONENT_ID:2009","CREATININE","F",0,30,0.3,1 "KUH|COMPONENT_ID:2009","CREATININE","F",30,256,0.2,0.4 "KUH|COMPONENT_ID:2009","CREATININE","F",256,4608,0.3,1 -"KUH|COMPONENT_ID:2009","CREATININE","F",4608,25600,0.4,1 +"KUH|COMPONENT_ID:2009","CREATININE","F",4608,36500,0.4,1 "KUH|COMPONENT_ID:2009","CREATININE","M",0,30,0.3,1 "KUH|COMPONENT_ID:2009","CREATININE","M",30,256,0.2,0.4 "KUH|COMPONENT_ID:2009","CREATININE","M",256,4608,0.3,1 -"KUH|COMPONENT_ID:2009","CREATININE","M",4608,25600,0.4,1.24 +"KUH|COMPONENT_ID:2009","CREATININE","M",4608,36500,0.4,1.24 "KUH|COMPONENT_ID:2012","GLUCOSE, POC","F",0,1,45,80 "KUH|COMPONENT_ID:2012","GLUCOSE, POC","F",1,30,50,80 -"KUH|COMPONENT_ID:2012","GLUCOSE, POC","F",30,25600,70,100 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","F",30,36500,70,100 "KUH|COMPONENT_ID:2012","GLUCOSE, POC","M",0,1,45,80 "KUH|COMPONENT_ID:2012","GLUCOSE, POC","M",1,30,50,80 -"KUH|COMPONENT_ID:2012","GLUCOSE, POC","M",30,25600,70,100 -"KUH|COMPONENT_ID:2320","HDL","F",0,25600,40, -"KUH|COMPONENT_ID:2320","HDL","M",0,25600,40, +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","M",30,36500,70,100 +"KUH|COMPONENT_ID:2320","HDL","F",0,36500,40, +"KUH|COMPONENT_ID:2320","HDL","M",0,36500,40, "KUH|COMPONENT_ID:3004","HEMATOCRIT","F",0,30,51,65 "KUH|COMPONENT_ID:3004","HEMATOCRIT","F",30,180,38,50 "KUH|COMPONENT_ID:3004","HEMATOCRIT","F",180,1024,30,42 "KUH|COMPONENT_ID:3004","HEMATOCRIT","F",1024,3584,35,46 -"KUH|COMPONENT_ID:3004","HEMATOCRIT","F",3584,25600,36,45 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","F",3584,36500,36,45 "KUH|COMPONENT_ID:3004","HEMATOCRIT","M",0,30,51,65 "KUH|COMPONENT_ID:3004","HEMATOCRIT","M",30,180,38,50 "KUH|COMPONENT_ID:3004","HEMATOCRIT","M",180,1024,30,42 "KUH|COMPONENT_ID:3004","HEMATOCRIT","M",1024,3584,36,46 -"KUH|COMPONENT_ID:3004","HEMATOCRIT","M",3584,25600,40,50 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","M",3584,36500,40,50 "KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",0,30,17.5,21.5 "KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",30,180,12.8,15.5 "KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",180,1024,10,14 "KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",1024,3584,11,14 -"KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",3584,25600,12,15 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",3584,36500,12,15 "KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",0,30,17.5,21.5 "KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",30,180,12.8,15.5 "KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",180,1024,10,14 "KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",1024,3584,11,14 -"KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",3584,25600,13.5,16.5 -"KUH|COMPONENT_ID:2034","HEMOGLOBIN A1c","F",0,25600,4,6 -"KUH|COMPONENT_ID:2034","HEMOGLOBIN A1c","M",0,25600,4,6 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",3584,36500,13.5,16.5 +"KUH|COMPONENT_ID:2034","HEMOGLOBIN A1c","F",0,36500,4,6 +"KUH|COMPONENT_ID:2034","HEMOGLOBIN A1c","M",0,36500,4,6 From 2d75dfd45fded088a99b6404d255a47052acf581 Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 22 Jan 2018 10:14:53 -0600 Subject: [PATCH 313/507] Baseline Luigi implementation in i2p-transform. Directory structure was revised, Luigi support files were copied from Heron, Oracle and Luigi libraries were updated and spot fixes were performed in Python code. --- CONTRIBUTING.md | 132 ++++ Oracle/PCORNetInit.sql | 3 +- Oracle/epic_facts_load.sql | 92 +++ Oracle/epic_flowsheets_transform.sql | 577 +++++++++++++++++ Oracle/etl_tests_init.sql | 84 +++ Oracle/migrate_fact_upload.sql | 22 + README.md | 56 ++ client.cfg | 58 ++ epic_flowsheets.py | 192 ++++++ etl_tasks.py | 899 +++++++++++++++++++++++++++ eventlog.py | 152 +++++ i2p_tasks.py | 11 + logging.cfg | 83 +++ param_val.py | 37 ++ pythonjsonlogger/__init__.py | 0 pythonjsonlogger/jsonlogger.py | 141 +++++ requirements.txt | 55 ++ script_lib.py | 313 ++++++++++ setup.cfg | 49 ++ sql-style.xml | 453 ++++++++++++++ sql_syntax.py | 238 +++++++ stubs/cx_Oracle.pyi | 429 +++++++++++++ stubs/luigi/__init__.pyi | 20 + stubs/luigi/configuration.pyi | 26 + stubs/luigi/contrib/__init__.pyi | 4 + stubs/luigi/contrib/sqla.pyi | 43 ++ stubs/luigi/event.pyi | 15 + stubs/luigi/interface.pyi | 37 ++ stubs/luigi/local_target.pyi | 35 ++ stubs/luigi/parameter.pyi | 131 ++++ stubs/luigi/rpc.pyi | 26 + stubs/luigi/target.pyi | 47 ++ stubs/luigi/task.pyi | 107 ++++ stubs/luigi/tools/__init__.pyi | 4 + stubs/luigi/tools/range.pyi | 86 +++ stubs/sqlalchemy/__init__.pyi | 124 ++++ stubs/sqlalchemy/engine/__init__.pyi | 11 + stubs/sqlalchemy/engine/base.pyi | 39 ++ stubs/sqlalchemy/engine/result.pyi | 86 +++ stubs/sqlalchemy/engine/url.pyi | 27 + stubs/sqlalchemy/exc.pyi | 27 + stubs/sqlalchemy/sql/__init__.pyi | 0 stubs/sqlalchemy/sql/elements.pyi | 25 + stubs/sqlalchemy/sql/expression.pyi | 7 + 44 files changed, 5001 insertions(+), 2 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 Oracle/epic_facts_load.sql create mode 100644 Oracle/epic_flowsheets_transform.sql create mode 100644 Oracle/etl_tests_init.sql create mode 100644 Oracle/migrate_fact_upload.sql create mode 100644 client.cfg create mode 100644 epic_flowsheets.py create mode 100644 etl_tasks.py create mode 100644 eventlog.py create mode 100644 i2p_tasks.py create mode 100644 logging.cfg create mode 100644 param_val.py create mode 100644 pythonjsonlogger/__init__.py create mode 100644 pythonjsonlogger/jsonlogger.py create mode 100644 requirements.txt create mode 100644 script_lib.py create mode 100644 setup.cfg create mode 100644 sql-style.xml create mode 100644 sql_syntax.py create mode 100644 stubs/cx_Oracle.pyi create mode 100644 stubs/luigi/__init__.pyi create mode 100644 stubs/luigi/configuration.pyi create mode 100644 stubs/luigi/contrib/__init__.pyi create mode 100644 stubs/luigi/contrib/sqla.pyi create mode 100644 stubs/luigi/event.pyi create mode 100644 stubs/luigi/interface.pyi create mode 100644 stubs/luigi/local_target.pyi create mode 100644 stubs/luigi/parameter.pyi create mode 100644 stubs/luigi/rpc.pyi create mode 100644 stubs/luigi/target.pyi create mode 100644 stubs/luigi/task.pyi create mode 100644 stubs/luigi/tools/__init__.pyi create mode 100644 stubs/luigi/tools/range.pyi create mode 100644 stubs/sqlalchemy/__init__.pyi create mode 100644 stubs/sqlalchemy/engine/__init__.pyi create mode 100644 stubs/sqlalchemy/engine/base.pyi create mode 100644 stubs/sqlalchemy/engine/result.pyi create mode 100644 stubs/sqlalchemy/engine/url.pyi create mode 100644 stubs/sqlalchemy/exc.pyi create mode 100644 stubs/sqlalchemy/sql/__init__.pyi create mode 100644 stubs/sqlalchemy/sql/elements.pyi create mode 100644 stubs/sqlalchemy/sql/expression.pyi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a8e9f6b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,132 @@ +## Design: ETL Tasks and SQL Scripts + +The main tasks are in: + + - *epic_flowsheets* -- ETL tasks for Epic Flowsheets to i2b2 + +which is based on a design in: + + - *etl_tasks* -- Source-agnostic Luigi ETL Task support + - *script_lib* -- library of SQL scripts + - *sql_syntax* -- break SQL scripts into statements, etc. + +Tasks such as `epic_flowsheets.FlowsheetViews` are based on SQL +scripts such as `sql_scripts/epic_flowsheets_transform.sql` wrapped in +a `etl_tasks.SqlScriptTask`. + + +### SQL Script Library Design, Style and Conventions + +Each script should start with a header comment and some +dependency-checking queries. See `script_lib.py` for details. + +SQL should be written in lowercase, indented 2 spaces, 120 maximum line +length. More details are in the evolving `sql-style.xml` sqldeveloper +style profile. + + - *ISSUE*: sqldeveloper 3 vs. 4 style files? + +See also notes on value enumerations in the header of `sql_scripts/cms_keys.pls`. +*TODO*: port those notes from GROUSE. + + +## Python doctest for story telling and unit testing + +Each python module header should tell a story using [doctest][], +i.e. examples that are also unit tests. + +You can run them one module at a time: + + (luigi)$ python -m doctest script_lib.py -v + Trying: + Script.migrate_fact_upload.title + Expecting: + 'append data from a workspace table.' + ... + 22 tests in 13 items. + 22 passed and 0 failed. + Test passed. + +Or install [nose][] and run all modules at once: + + (grouse-etl)% nosetests --with-doctest + ...................... + ---------------------------------------------------------------------- + Ran 22 tests in 0.658s + + OK + +[doctest]: http://docs.python.org/2/library/doctest.html +[nose]: https://pypi.python.org/pypi/nose/ + + +## Python code style + +We appreciate object capability discipline and the "don't call us, +we'll call you" style that facilitates unit testing with mocks. + + - *ISSUE*: Luigi's design doesn't seem to support this idiom. + Constructors are implict and tasks parameters have to be + serializable, which works against the usual closure + object pattern. Also, the task cache is global mutable + state. + +We avoid mutable state, preferring functional style. + + - *NOTE*: PEP8 tools warn against assinging a lambda to a name, + suggesting `def` instead. We're fine with it; hence + `ignore = E731` in `setup.cfg`. + + +We follow PEP8. The first line of a module or function docstring +should be a short description; if it is blank, either the item is in +an early stage of development or the name and doctests are supposed to +make the purpose obvious. + + - *NOTE*: with static type annotations, the 79 character line + length limit is awkward; hence we use 99 in `setup.cfg`. + + - *ISSUE*: Dan didn't realize until recently that PEP8 recommends + triple double quotes over triple single quotes for + docstrings. He's in the habit of using single quotes + to minimize use of the shift key. + + +## Checking the code + +Once dependencies in `requirements.txt` are satisfied, code should +pass tests, style checks, and static type checking: + + $ nosetests && flake8 . && mypy . + +_tested with mypy-0.521_ + +### Checking in emacs + +To check with `M-x compile` in emacs, first use `M-x pyvenv-activate` +from the [pyvenv][] package. + +To check continuously as you edit, use [flycheck][] and activate +likewise. + +[pyvenv]: https://melpa.org/#/pyvenv +[pyvent]: https://melpa.org/#/flycheck + + +## Luigi Troubleshooting + +**ISSUE**: why won't luigi find modules in the current directory? + Use `PYTHONPATH=. luigi ...` if necessary. + +Most diagnostics are self-explanatory; `etl_tasks` includes +`SQLScriptError` and `ConnectionProblem` exception classes intended to +improve diagnostics + +One challenging diagnostic is: + + RuntimeError: Unfulfilled dependency at run time: DiagnosesLoad_oracle___dconnol_CMS_DEID_SAMPLE_1438246788671_bd6231c982 + +It seems to indicate that the `.complete()` test on a required task +fails even after that task has been `.run()`. For example, the `select +count(*)` completion test in a load script might have failed because +of incorrect join constraints. diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index 934c863..6149995 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -1007,5 +1007,4 @@ CREATE TABLE harvest( REFRESH_DEATH_CAUSE_DATE date NULL ) / - - +SELECT 1 FROM HARVEST diff --git a/Oracle/epic_facts_load.sql b/Oracle/epic_facts_load.sql new file mode 100644 index 0000000..6bff120 --- /dev/null +++ b/Oracle/epic_facts_load.sql @@ -0,0 +1,92 @@ +/** epic_facts_load.sql - Load observations from Epic Clarity. + +Copyright (c) 2012-2017 University of Kansas Medical Center +part of the HERON* open source codebase; see NOTICE file for license details. +* http://informatics.kumc.edu/work/wiki/HERON + +see also epic_dimensions_load.sql, + http://informatics.kumc.edu/work/wiki/HeronLoad +*/ + +/* Check for id repository, uploader service tables */ +select * from NightHeronData.observation_fact where 1 = 0; +select * from NightHeronData.upload_status where 1 = 0; + +/* We're wasting our time if an upload_status record isn't in place. */ +select case count(*) when 1 then 1 else 1/0 end as upload_status_exists from ( +select * from NightHeronData.upload_status up +where up.upload_id = :upload_id +); + + +create table observation_fact_&&upload_id as +select * from NightHeronData.observation_fact where 1=0; + +insert /*+ append */ into observation_fact_&&upload_id ( + patient_num, encounter_num, sub_encounter, + concept_cd, + provider_id, + start_date, + modifier_cd, + instance_num, + valtype_cd, + tval_char, + nval_num, + valueflag_cd, + units_cd, + end_date, + location_cd, + update_date, + import_date, upload_id, download_date, sourcesystem_cd) +select pmap.patient_num, + coalesce(emap.encounter_num, -abs(ora_hash(to_char(f.start_date, 'YYYYMMDD') || f.patient_ide))), -- TODO: pat_day func + f.encounter_ide, + f.concept_cd, + f.provider_id, + f.start_date, + f.modifier_cd, + f.instance_num, + f.valtype_cd, + f.tval_char, + f.nval_num, + f.valueflag_cd, + f.units_cd, + f.end_date, + f.location_cd, + f.update_date, + sysdate, up.upload_id, :download_date, up.source_cd +from &&epic_fact_view f + join NightHeronData.patient_mapping pmap + on pmap.patient_ide = f.patient_ide + and pmap.patient_ide_source = :pat_source_cd + and pmap.patient_ide_status = 'A' -- TODO: (select active from i2b2_status) + left join NightHeronData.encounter_mapping emap + on emap.encounter_ide = f.encounter_ide + and emap.encounter_ide_source = :enc_source_cd + and emap.encounter_ide_status = 'A' + , NightHeronData.upload_status up +where up.upload_id = :upload_id + and f.patient_ide between :pat_id_lo and :pat_id_hi + &&log_fact_exceptions +; +commit +; + + +/* For this upload of data, check primary key constraints. +This could perhaps be factored out of all the load scripts into i2b2_facts_deid.sql, +at the cost of slowing down the select count(*) for summary stats, below. + +TODO: check key constraint + +alter table observation_fact_&&upload_id + enable constraint observation_fact_pk + -- TODO: log errors ... ? #2117 + ; + */ + + +select count(*) +from NightHeronData.upload_status +where transform_name = :task_id and load_status = 'OK'; + diff --git a/Oracle/epic_flowsheets_transform.sql b/Oracle/epic_flowsheets_transform.sql new file mode 100644 index 0000000..fc40305 --- /dev/null +++ b/Oracle/epic_flowsheets_transform.sql @@ -0,0 +1,577 @@ +/** epic_flowsheets_transform.sql -- prepare to load Epic EMR flowsheet facts + +Copyright (c) 2012 University of Kansas Medical Center +part of the HERON* open source codebase; see NOTICE file for license details. +* http://informatics.kumc.edu/work/wiki/HERON + + * See also epic_flowsheets_multiselect.sql, epic_etl.py, epic_facts_load.sql + * + * References: + * + * Lesson 4: Flowsheet Data + * Clarity Data Model - EpicCare Inpatient Spring 2008 Training Companion + * https://userweb.epic.com/epiclib/epicdoc/EpicWiseSpr08/Clarity/Clarity%20Training%20Companions/CLR209%20Data%20Model%20-%20EpicCare%20Inpatient/04TC%20Flowsheet%20Data.doc + */ + +select test_name from etl_tests where 'dep' = 'etl_tests_init.sql'; + +-- Check that this is the ID instance +select flo_meas_id from CLARITY.ip_flwsht_meas where 1=0; + +-- to see the time detail... +alter session set NLS_DATE_FORMAT = 'YYYY-MM-DD HH:MI'; + +-- define heron_obs_sample = sample (0.001, 1234) + +create or replace view etl_test_domain_flowsheets as +select 'Flowsheets' test_domain from dual; + +insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) +with test_key as ( + select test_domain, + 'val_type_seen_before' test_name from etl_test_domain_flowsheets + ) +, test_values as ( + select count(*) test_value, test_key.* from ( + select distinct zcvt.name + from CLARITY.ip_flwsht_meas ifm + join CLARITY.ip_flo_gp_data ifgd + on ifm.flo_meas_id= ifgd.flo_meas_id + join CLARITY.zc_val_type zcvt on ifgd.val_type_c = zcvt.val_type_c + where zcvt.name not in ( + 'Blood Pressure', 'Category Type', 'Concentration', 'Custom List', + 'Date', 'Dose', 'Height', 'Numeric Type', 'Patient Height', + 'Patient Weight', 'Rate', 'String Type', 'Temperature', 'Time', 'Weight') + ), test_key + ) +select test_value, test_domain, test_name, sq_result_id.nextval, sysdate +from test_values +; + + +/*************** + * flo_meas_type -- per-flo_meas_id transformation + +Starting with relevant columns from ip_flo_gp_data, +determine i2b2 valtype_cd, UNITS_CD. + +See also i2b2_facts_deid.sql regarding identified/de-identified data +flag in valtype_cd. + +Values for height, weight, and temperature are (for reasons lost to history) +converted to SI units. + +Also, trivially set up MODIFIER_CD, CONFIDENCE_NUM. + +TODO: consider migrating to CSV spreadsheet. Too bad SQL doesn't + have relation constants a la R read.table() or JSON. + + */ +-- compute i2b2 valtype_cd for each flo_meas_id +create or replace view flo_meas_type as +select ifgd.flo_meas_id, ifgd.disp_name + , ifgd.multi_sel_yn + , ifgd.record_state_c + , zcvt.name val_type + , case + when zcvt.name in ( + 'Numeric Type', + 'Blood Pressure', 'Temperature', 'Time', + 'Patient Weight', 'Weight', + 'Patient Height', 'Height') then 'N' + when zcvt.name in ('Custom List', 'Category Type') then '@' + -- only add ID information to nightheron + when zcvt.name in ('String Type') then 'Ti' + when zcvt.name in ('Date') then 'D' + else null -- val_type_seen_before test keeps these bounded + end valtype_cd + , 'KUH|FLO_MEAS_ID:' || ifgd.flo_meas_id CONCEPT_CD + + -- Convert to SI units per 2011 design. + , case + when zcvt.name in ( + /* Epic documentation doesn't give units for 'Weight', + so we've determined emperically that it's oz. + See also mean Birth Weight test based on flo_stats_num + in epic_flowsheets_multiselect.sql */ + 'Patient Weight', 'Weight') then 1.0 / 16.0 / 2.2 -- 16 oz/lb; 2.2kg/lb + when zcvt.name in ( + 'Patient Height', 'Height') then 2.54 -- 2.54 cm/in + when zcvt.name in ( + 'Temperature') then 5/9 -- F to C + else 1 + end scale + , case + when zcvt.name in ( + 'Temperature') then -32 -- F to C + else 0 + end bias + , case + when zcvt.name in ( + 'Patient Weight', 'Weight') then 'kg' + when zcvt.name in ( + 'Temperature') then 'C' + when zcvt.name in ( + 'Time') then 's' + when zcvt.name in ( + 'Patient Height', 'Height') then 'cm' + when zcvt.name in ( + 'Blood Pressure') then 'mmHg' + else ifgd.units + end UNITS_CD + , '@' MODIFIER_CD + , to_number(null) CONFIDENCE_NUM +from CLARITY.ip_flo_gp_data ifgd +left join CLARITY.zc_val_type zcvt on ifgd.val_type_c = zcvt.val_type_c +; + + +/*************** + * flowsheet_day -- per-patient-day transformation + +Starting with ip_flwsht_rec, compute ENCOUNTER_IDE, PATIENT_IDE +(to be mixed with patient_mapping and encounter_mapping in epic_facts_load.sql). + +Trivially set LOCATION_CD. + + */ +create or replace view flowsheet_day as +select ifr.record_date + , ifr.pat_id + , ifr.fsd_id + , ifr.INPATIENT_DATA_ID + -- i2b2 equivalents common to many/all datatypes + + -- patient day; see patient_day_visit view + , TO_CHAR(ifr.record_date,'YYYYMMDD') || ifr.pat_id + ENCOUNTER_IDE + , ifr.pat_id PATIENT_IDE + , '@' LOCATION_CD +from CLARITY.ip_flwsht_rec ifr +; + + +/*************** + * flowsheet_obs -- per-measurement transformation common to all datatypes + +Starting with ip_flwsht_meas, compute instance_num, +start_date, end_date, update_date. + +Filter out rows where meas_value is null. + +Trivially (for now) set PROVIDER_ID, valueflag_cd. + + */ +create or replace view flowsheet_obs as +select /*+ parallel(16) */ -- ISSUE: parameterize parallel_degree? + ifm.fsd_id + , ifm.line + , ifm.flo_meas_id + , ifm.meas_value + , ifm.recorded_time + , ifm.entry_time + -- per-measurement conversion to i2b2 terms + , '@' PROVIDER_ID -- todo: use ifm.TAKEN_USER_ID + , recorded_time START_DATE + , ifm.fsd_id * 100000 + ifm.line instance_num + , '@' valueflag_cd -- TODO #3850: [H]igh/[L]ow/[A]bnormal based on ifm.ABNORMAL_C + , recorded_time END_DATE + , entry_time UPDATE_DATE +from CLARITY.ip_flwsht_meas ifm +where meas_value is not null +; + + +/******************** + * flowsheet_num_data - handle number syntax + +Starting with flowsheet_obs, split diastolic and systolic into separate rows, +flag (but don't filter) illegal numerals, and convert the rest to_number(). + +Values such as 2.22222222222222E+22 don't fit in nval_num, +which is declared NUMBER(18,5). Let's throw out values using E notation. +The `numerals_filtered` test (in epic_flowsheets_multiselect) verifies that +we're not throwing away more than a handful. + +Stats in epic_flowsheets_multiselect.sql are based on this view, +before we join with per-patient-day or per-flow-measure info. + + */ +create or replace view flowsheet_num_data as +with +parts as ( + select flo_meas_id, bp_part, pattern + from flo_meas_type fmt + left join ( + select 'Blood Pressure' val_type, '_DIASTOLIC' bp_part, '\d+$' pattern from dual +union all select 'Blood Pressure' val_type, '_SYSTOLIC' bp_part, '^\d+' pattern from dual + ) parts on parts.val_type = fmt.val_type + where fmt.valtype_cd like 'N%' +) +, numerals as ( + select fsd_id, line + , obs.flo_meas_id + , bp_part + , meas_value + , recorded_time + , entry_time + , case + when parts.pattern is not null + then regexp_substr(meas_value, parts.pattern) + else meas_value + end numeral + , START_DATE + , END_DATE + , instance_num + , UPDATE_DATE + , PROVIDER_ID + , valueflag_cd + from flowsheet_obs obs + join parts on parts.flo_meas_id = obs.flo_meas_id +) +, num_check as ( + select numerals.* + , case + when numeral is null then null + when regexp_like(numeral, '^-?\d*(\.\d+)?$') then 1 + else 0 + end ok + from numerals +) + select num_check.* + , case + when ok = 1 then to_number(numeral) + else null + end meas_value_num + from num_check +; + + +/************** + * numerictypeflows - Numeric, Blood Pressure, Weight, etc. + +Build concept_cd using blood pressure suffix; apply units bias and scale; +trivially set TVAL_CHAR. + +Join flowsheet_num_data with flo_meas_type and flowsheet_day to fill in +remaining observation_fact style fields. + + */ +create or replace view numerictypeflows as +select ENCOUNTER_IDE + , PATIENT_IDE + , 'KUH|FLO_MEAS_ID:' || num_data.flo_meas_id || bp_part concept_cd + , PROVIDER_ID + , START_DATE + , MODIFIER_CD + , instance_num + , valtype_cd + , 'E' TVAL_CHAR -- i2b2 doc says 'EQ' but demo data says 'E' + , case when fmt.bias <> 0 or fmt.scale <> 1 + then + -- Round so as not to imply more precision than was measured. + round((meas_value_num + fmt.bias) * fmt.scale, 3) + else + meas_value_num + end nval_num + , valueflag_cd + , UNITS_CD + , END_DATE + , LOCATION_CD + , CONFIDENCE_NUM + , UPDATE_DATE +from flowsheet_num_data num_data +join flo_meas_type fmt on fmt.flo_meas_id = num_data.flo_meas_id +join flowsheet_day fsd on fsd.fsd_id = num_data.fsd_id +where num_data.ok = 1 +; + +-- select * from numerictypeflows; + + +/*********** + * datemeasureflows - for value_type_name "Date" + * + * STRANGE! Syntax is EITHER: + * - a number of days since Dec 31, 1840 or + * - a date in 'MM/DD/YYYY' format + */ +create or replace view datemeasureflows as +Select + ENCOUNTER_IDE + , PATIENT_IDE + , CONCEPT_CD + , PROVIDER_ID + , START_DATE + , MODIFIER_CD + , instance_num + , 'D' VALTYPE_CD, + case + when substr(meas_value , 1, instr(meas_value , '/') -1 ) is NULL + then to_char(to_date('12/31/1840', 'MM/DD/YYYY') + meas_value, 'YYYY-MM-DD') + else to_char(to_date(meas_value, 'MM/DD/YYYY'), 'YYYY-MM-DD') + end TVAL_CHAR + , null NVAL_NUM + , valueflag_cd + , UNITS_CD + , END_DATE + , LOCATION_CD + , CONFIDENCE_NUM + , UPDATE_DATE +from flowsheet_obs obs +join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id +join flowsheet_day fsd on fsd.fsd_id = obs.fsd_id +where valtype_cd like 'D%'; + + +/* todo:: test date-shifting when VALUE_TYPE_NAME='Date' + e.g. 11568 or 700 PLACEMENT DATE (#299) */ + + +/***************** + * idstringtypeflows -- free text + +For measures identifed (in flo_meas_type) as free text, set tval_char. + +Trivially set nval_num. + */ +create or replace view idstringtypeflows as +select + fsd.ENCOUNTER_IDE + , fsd.PATIENT_IDE + , fmt.CONCEPT_CD + , obs.PROVIDER_ID + , obs.START_DATE + , fmt.MODIFIER_CD + , obs.instance_num + , fmt.VALTYPE_CD + , obs.meas_value TVAL_CHAR + , to_number(null) NVAL_NUM + , obs.valueflag_cd + , fmt.UNITS_CD + , obs.END_DATE + , fsd.LOCATION_CD + , fmt.CONFIDENCE_NUM + , obs.UPDATE_DATE +from flowsheet_obs obs +join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id +join flowsheet_day fsd on fsd.fsd_id = obs.fsd_id +where valtype_cd like 'T%'; + + +/***************** + * deidstringtypeflows -- deid-ed free text + +For measures identifed (in flo_meas_type) as free text, and have corresponding +de-identified meas_values. + +As an intermediate step create `approved_deid_flowsheets` containing all of the +de-identified free text notes that have been approved for inclusion. + +Trivially set nval_num. + */ + + +whenever sqlerror continue; +drop table approved_deid_flowsheets; +whenever sqlerror exit; + + +create table approved_deid_flowsheets ( + fsd_id number + , line number + , flo_meas_id number + , meas_value varchar(4000) + , primary key (fsd_id, line)); + + +whenever sqlerror continue; +drop table approved_flo_meas_ids; +-- Oracle gave really bad performance while `deidentify.deid_flowsheet_approval` +-- was a view. +create table approved_flo_meas_ids as +select * from deidentify.deid_flowsheet_approval where approved_to_deid = 1; + +insert into approved_deid_flowsheets +(fsd_id, line, flo_meas_id, meas_value) +select fsd_id, line, flo_meas_id, meas_value +from ( + -- `fsd_id` and `line` are extracted from `flo_notes.id` + -- see: heron_staging/deidentify/Flowsheets-DEID-Stage.sql + select to_number(substr(to_char(id), + 1, + length(to_char(id))-5)) as fsd_id + , to_number(substr(id, -5)) as line + , to_number(note_id) as flo_meas_id + , deid_note_text as meas_value + from deidentify.flo_notes) deid + -- Include only meas_values that have flo_meas_id approved. +where exists (select * from approved_flo_meas_ids af + where deid.flo_meas_id = af.flo_meas_id + ); +whenever sqlerror exit; + + +insert into etl_test_values (test_value, test_domain, test_name, result_id, + result_date) +with test_key as ( + select test_domain, + 'staged_deid_flowsheet' test_name from etl_test_domain_flowsheets + ) +, test_values as ( + select count(*) test_value, test_key.* from approved_deid_flowsheets, test_key + ) +select test_value, test_domain, test_name, sq_result_id.nextval, sysdate +from test_values +; + +-- Did `staged_deid_flowsheet` fail? First check to see if the necessary +-- staged tables exist. + +-- select * from deidentify.flo_notes; +-- select * from deidentify.deid_flowsheet_approval; + +create or replace view deidstringtypeflows as +select + fsd.ENCOUNTER_IDE + , fsd.PATIENT_IDE + , fmt.CONCEPT_CD + , obs.PROVIDER_ID + , obs.START_DATE + , fmt.MODIFIER_CD + , obs.instance_num + , 'Td' as valtype_cd + , deid.meas_value TVAL_CHAR + , to_number(null) NVAL_NUM + , obs.valueflag_cd + , fmt.UNITS_CD + , obs.END_DATE + , fsd.LOCATION_CD + , fmt.CONFIDENCE_NUM + , obs.UPDATE_DATE +from flowsheet_obs obs +join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id +join flowsheet_day fsd on fsd.fsd_id = obs.fsd_id +join approved_deid_flowsheets deid + on deid.fsd_id = obs.fsd_id + and deid.line = obs.line +where valtype_cd like 'T%'; + + +/******************** + * flowsheet_nom_data - nominal data, including multi select + +Stats in epic_flowsheets_multiselect.sql are based on this view, +before we join with per-patient-day or per-flow-measure info. + +The flow_measure_multi table (eventually) maps 'x;y;z' multi-select values +to the constituent 'x', 'y', and 'z' values. Create the table so we can +refer to it, but leave populating it to epic_flowsheets_multiselect.sql. + + */ +whenever sqlerror continue; +drop table flow_measure_multi; +whenever sqlerror exit; +create table flow_measure_multi as +select fm.meas_value + , 0 val_ix + , fm.meas_value choice_label +from flowsheet_obs fm +where 1=0; + +create or replace view flowsheet_nom_data as + select fsd_id, obs.line + , obs.flo_meas_id + , fmt.multi_sel_yn + , obs.meas_value + , fmm.val_ix + , fmm.choice_label multi_choice_label + , coalesce(fmm.choice_label, obs.meas_value) choice_label + , recorded_time + , entry_time + , START_DATE + , END_DATE + , UPDATE_DATE + , PROVIDER_ID + , valueflag_cd + from flowsheet_obs obs + join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id + and fmt.valtype_cd = '@' -- valtype_cd for nominal data + left join flow_measure_multi fmm + -- Try not to look in flow_measure_multi unless + -- except when we have multi_sel_yn and a ';' . + on fmm.meas_value = case + when fmt.multi_sel_yn = '1' + and obs.meas_value like '%;%' + then obs.meas_value + else null + end +; + +-- "forward reference" flo_stats_nom table for use in flowsheets_concepts.sql +whenever sqlerror continue; +drop table flo_stats_nom; +whenever sqlerror exit; + +create table flo_stats_nom as +select flo_meas_id, meas_value choice_label + , 0 qty + , 0 tot + , 99.9 pct + , date '2001-01-01' recorded_time_min + , date '2001-01-01' recorded_time_max +from flowsheet_nom_data +where 1 = 0 +; + + +/************* + * selectflows -- Custom List nominal observations + +Compute concept_cd using flo_meas_id and ip_flo_cust_list.line where available; +fall back to hash of choice label. + +Trivially set tval_char, nval_num. + */ + +create or replace view selectflows as +Select ENCOUNTER_IDE + , PATIENT_IDE + , (case when ifcl.line is not null + then 'KUH|FLO_MEAS_ID+LINE:' || obs.flo_meas_id || '_' || ifcl.line + else 'KUH|FLO_MEAS_ID+hash:' || obs.flo_meas_id || '_' || ora_hash(obs.choice_label) + end) CONCEPT_CD + , PROVIDER_ID + , START_DATE + , MODIFIER_CD + -- Handle cases such as 'Sinus Tachycardia;ST' + -- where multiple choices map to the same ifcl.line + -- by including the val_ix (1, 2) in instance_num. + , (obs.fsd_id * 100000 + obs.line) * 10 + coalesce(val_ix, 0) instance_num + , VALTYPE_CD + , '@' TVAL_CHAR + , to_number(null) NVAL_NUM + , valueflag_cd + , UNITS_CD + , END_DATE + , LOCATION_CD + , CONFIDENCE_NUM + , UPDATE_DATE +from flowsheet_nom_data obs +join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id +join flowsheet_day fsd on fsd.fsd_id = obs.fsd_id +left join CLARITY.ip_flo_cust_list ifcl + on ifcl.flo_meas_id = obs.flo_meas_id + and (obs.choice_label = ifcl.abbreviation or + obs.choice_label = ifcl.custom_value) +where valtype_cd = '@'; + +-- select * from multiselectflows where rownum < 200; + + +/* complete check */ +create or replace view epic_flowsheets_txform_sql as +select &&design_digest design_digest from dual; + +select 1 up_to_date +from epic_flowsheets_txform_sql where design_digest = &&design_digest; diff --git a/Oracle/etl_tests_init.sql b/Oracle/etl_tests_init.sql new file mode 100644 index 0000000..21db271 --- /dev/null +++ b/Oracle/etl_tests_init.sql @@ -0,0 +1,84 @@ +/* etl_tests_init -- Initialize ETL tests + +Copyright (c) 2015 University of Kansas Medical Center +part of the HERON* open source codebase; see NOTICE file for license details. +* http://informatics.kumc.edu/work/wiki/HERON + +References: + * KUMC Informatics ticket #2781 +*/ + +whenever sqlerror continue; +drop table etl_tests; +whenever sqlerror exit; + +create table etl_tests ( + /* Description of test */ + test_domain varchar(256) not null -- Such as top-level folder in HERON that's affected (Frontiers Registry...) + , test_name varchar(1024) not null -- User friendly name (hictr_mrn_leading_zero...) + /* Pass/Fail criteria - can define one or both of the following - INCLUSIVE */ + , lower_bound number null + , upper_bound number null + , units varchar(256) null -- What are we counting? (patients, facts, concepts, etc.) + , test_description varchar(4000) null -- More detailed description - how is the end-user affected if this test fails? + ); + +alter table etl_tests add constraint pk_etl_tests primary key (test_domain, test_name); + +-- Table to store test values (inserted during ETL) +whenever sqlerror continue; +drop table etl_test_values; +whenever sqlerror exit; + +create table etl_test_values as + select test_domain, test_name from etl_tests where 1=0; + +alter table etl_test_values + add (test_value number not null + , detail_char_1 varchar(256) + , detail_char_2 varchar(256) + , detail_num_1 number + , detail_num_2 number + , test_note varchar(256) + , result_id number not null + , result_date date not null); + +alter table etl_test_values add constraint pk_etl_test_results primary key (result_id); + +-- I wish we had "auto increment". We can use a trigger but then we need PL/SQL. +whenever sqlerror continue; +drop sequence sq_result_id; +whenever sqlerror exit; +create sequence sq_result_id; + +/* Insert some example tests for debugging. */ +-- Example test that doesn't have something in the CSV +insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) +with test_key as ( + select 'test' test_domain, + 'test_not_found_in_csv' test_name from dual + ) +select 666 test_value, test_key.*, sq_result_id.nextval, sysdate from test_key; + +-- Example successful test +insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) +with test_key as ( + select 'test' test_domain, + 'test_success' test_name from dual + ) +select 1 test_value, test_key.*, sq_result_id.nextval, sysdate from test_key; + +-- Example test with no bounds specified +insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) +with test_key as ( + select 'test' test_domain, + 'test_no_bound' test_name from dual + ) +select 1 test_value, test_key.*, sq_result_id.nextval, sysdate from test_key; + + +/* complete check */ +select 1 complete +from (select test_name from etl_test_values where 1=0 + union all select * from dual -- guarantee 1 row + ); diff --git a/Oracle/migrate_fact_upload.sql b/Oracle/migrate_fact_upload.sql new file mode 100644 index 0000000..9dc8855 --- /dev/null +++ b/Oracle/migrate_fact_upload.sql @@ -0,0 +1,22 @@ +/** migrate_fact_upload - append data from a workspace table. +*/ + +insert /*+ parallel(&¶llel_degree) append */ into &&I2B2STAR.observation_fact +select * from &&workspace_star.observation_fact_&&upload_id ; + +insert into &&I2B2STAR.upload_status +select * from &&workspace_star.upload_status where upload_id = &&upload_id +-- Handle the case where workspace_star and I2B2STAR are the same. +and upload_id not in (select upload_id from &&I2B2STAR.upload_status) +; + +update &&I2B2STAR.upload_status set load_status = 'OK' +where upload_id = &&upload_id; + +commit ; + + +select 1 complete +from "&&I2B2STAR".upload_status +where upload_id = &&upload_id +; diff --git a/README.md b/README.md index 2730593..1b534c8 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,58 @@ # i2p-transform i2b2 to PCORnet Common Data Model Transformation - requires i2b2 PCORnet ontology + +# HERON ETL to i2b2 + +We (*@@TODO aim to*) transform the data from a number of sources and +load it into an i2b2 data repository, following the +[i2b2 Data Repository (CRC) Cell Design Document][CRC]. + +[CRC]: https://www.i2b2.org/software/files/PDF/current/CRC_Design.pdf + +## Usage + +To build flowsheet transformation views, +once installation and configuration (below) is done, invoke the +`FlowsheetViews` task following [luigi][] conventions: + + luigi --module epic_flowsheets FlowsheetViews + +In more detail: + + PYTHONPATH=. LUIGI_CONFIG_PATH=heron-test.cfg luigi \ + --local-scheduler \ + --module epic_flowsheets FlowsheetViews + +*@@TODO: actual fact loading* + +Then use `etl_tasks.MigratePendingUploads` to install the data in the +production i2b2 `observation_fact` table. Use +`@@TODO.PatientDimension` and `@@TODO.VisitDimLoad` to load the +`patient_dimension` and `visit_dimension` tables from +`observation_fact`. + +[luigi]: https://github.com/spotify/luigi + +Troubleshooting is discussed in [CONTRIBUTING][]. + +## Installation, Dependencies + +See `requirements.txt`. + +### luigid (optional) + + *@@TODO elaborate on this once we establish production conventions.* + + docker run --name luigid -p8082:8082 -v/STATE:/luigi/state -v/CONFIG:/etc/luigi -t stockport/luigid + + +## Configuration + +See [client.cfg](client.cfg). + +## Design and Development + +For design info and project coding style, see [CONTRIBUTING][]. + +[CONTRIBUTING]: CONTRIBUTING.md + diff --git a/client.cfg b/client.cfg new file mode 100644 index 0000000..1a925da --- /dev/null +++ b/client.cfg @@ -0,0 +1,58 @@ +# Usage: +# +# Copy client.cfg to your.cfg and set LUIGI_CONFIG_PATH=your.cfg. + +[core] +logging_conf_file = logging.cfg + +[ETLAccount] +# Account identifier in SQLAlchemy URL format. Be sure this includes +# all and only those details that identify the ETL +# account. Insignificant details such as password and tunnel port go +# in other parameters; including them here would result in +# inconsistent task identifiers. +# +# WRONG: +# oracle://username:password@localhost:5555/database_sid +# +# RIGHT: +account=oracle://grouse_etl_1@dbhost2/database_sid +passkey=GROUSE_ETL_1_ON_DBHOST2 +ssh_tunnel=localhost:4768 + + +[CLARITYExtract] +# When was it downloaded? +# e.g. suppose a jenkins download job was http://.../job/cms_syn_dl/8 +# and it finished Jul 30, 2015 8:54:25 AM. The we have: +download_date=1487378515445 + +# In order to get build dates from jenkins to luigi, i.e. +# from groovy to python, we use integers, since date interchange +# is a pain. +# +# Using the Jenkins API http://javadoc.jenkins.io/ +# A bit of groovy like this gets what we need: +# dlBuild?.timestamp.getTimeInMillis() + dlBuild?.duration + +# TODO: Split work into how many chunks by bene_id? +# bene_chunks = 64 + + +[NightHeronCreate] +# Where did we (or should we) create an i2b2 project? +# ref i2b2 sources: +# crc_create_datamart_oracle.sql +# crc_create_uploader_oracle.sql +star_schema = NIGHTHERONDATA + +# And what i2b2 project_id? +project_id = BlueHeron + +#[scheduler] +# ISSUE: task history? +#record_task_history = true + +[resources] +encounter_mapping=1 +patient_mapping=1 diff --git a/epic_flowsheets.py b/epic_flowsheets.py new file mode 100644 index 0000000..1466131 --- /dev/null +++ b/epic_flowsheets.py @@ -0,0 +1,192 @@ +'''epic_flowsheets -- ETL tasks for Epic Flowsheets to i2b2 +''' + +from typing import List + +import luigi +import sqlalchemy as sqla + +from etl_tasks import ( + DBAccessTask, I2B2ProjectCreate, I2B2Task, SourceTask, + SqlScriptTask, UploadTask, + DBTarget, SchemaTarget, UploadTarget +) +from script_lib import Script +from sql_syntax import Environment, Params +import param_val as pv + + +class CLARITYExtract(SourceTask, DBAccessTask): + download_date = pv.TimeStampParam(description='see client.cfg') + source_cd = pv.StrParam(default="Epic@kumed.com") + + # ISSUE: parameterize CLARITY schema name? + schema = 'CLARITY' + table_eg = 'patient' + + def _dbtarget(self) -> DBTarget: + return SchemaTarget(self._make_url(self.account), + schema_name=self.schema, + table_eg=self.table_eg, + echo=self.echo) + + +class NightHeronCreate(I2B2ProjectCreate): + pass + + +class NightHeronTask(I2B2Task): + '''Mix in identified star_schema parameter config. + ''' + @property + def project(self) -> I2B2ProjectCreate: + return NightHeronCreate() + + +class FromEpic(NightHeronTask): + '''Mix in source and substitution variables for Epic ETL scripts. + ''' + @property + def source(self) -> CLARITYExtract: + return CLARITYExtract() + + @property + def variables(self) -> Environment: + return self.vars_for_deps + + @property + def vars_for_deps(self) -> Environment: + # TODO: config = [] + # TODO: design = [] # TODO + return dict() + + +class FlowsheetViews(SqlScriptTask): + # TODO: subsume this in a load task + # TODO: patient survey using ntile() -- persistent worthwhile? + script = Script.epic_flowsheets_transform + + +class EpicDimensionsLoad(FromEpic, DBAccessTask): + '''Placeholder for heron_load/load_epic_dimensions.sql + ''' + @property + def transform_name(self) -> str: + return 'load_epic_dimensions' + + def complete(self) -> bool: + return self.output().exists() + + def output(self) -> luigi.Target: + return self._upload_target() + + def _upload_target(self) -> 'UploadTarget': + return UploadTarget(self._make_url(self.account), + self.project.upload_table, + self.transform_name, self.source, + echo=self.echo) + + def run(self) -> None: + raise NotImplementedError + + +class PatIdMapping(luigi.WrapperTask): + def requires(self) -> List[luigi.Task]: + # TODO: split pat_id mapping out of load_epic_dimensions.sql + return [EpicDimensionsLoad()] + + +class FSDMapping(luigi.WrapperTask): + def requires(self) -> List[luigi.Task]: + # TODO: split pat_id mapping out of load_epic_dimensions.sql + return [EpicDimensionsLoad()] + + +class EpicFactsLoadGroup(FromEpic, UploadTask): + epic_fact_view = pv.StrParam() + pat_id_lo = pv.StrParam() + pat_id_hi = pv.StrParam() + pat_group_num = pv.IntParam(significant=False) + pat_group_qty = pv.IntParam(significant=False) + pat_source_cd = pv.StrParam(default='Epic@kumed.com') + enc_source_cd = pv.StrParam() + + script = Script.epic_facts_load + + @property + def label(self) -> str: + return '{view} #{group}: {lo} to {hi}'.format( + view=self.epic_fact_view, group=self.pat_group_num, + lo=self.pat_id_lo, hi=self.pat_id_hi) + + def requires(self) -> List[luigi.Task]: + return UploadTask.requires(self) + [ + self.source, + FSDMapping(), + PatIdMapping(), + ] + + @property + def variables(self) -> Environment: + return dict(self.vars_for_deps, + epic_fact_view=self.epic_fact_view, + log_fact_exceptions='') + + def script_params(self) -> Params: + return dict(UploadTask.script_params(self), + pat_source_cd=self.pat_source_cd, + enc_source_cd=self.enc_source_cd, + pat_id_lo=self.pat_id_lo, + pat_id_hi=self.pat_id_hi) + + +class FlowsheetsLoad(FromEpic, DBAccessTask, luigi.WrapperTask): + pat_group_qty = pv.IntParam(default=5, significant=False) + enc_source_cd = pv.StrParam(default='Epic+pat_id_day@kumed.com') + + # issue: parameterize fact views? + fact_views = [ + 'numerictypeflows', + 'numerictypeflows', + 'datemeasureflows', + 'selectflows', + 'idstringtypeflows', + 'deidstringtypeflows', + ] + + pat_grp_q = ''' + select :group_qty grp_qty, group_num + , min(pat_id) pat_id_lo + , max(pat_id) pat_id_hi + from ( + select pat_id + , ntile(:group_qty) over (order by pat_id) as group_num + from ( + select /*+ parallel(20) */ distinct pat_id + from clarity.patient + where pat_id is not null -- help Oracle use the index + ) ea + ) w_ntile + group by group_num, :group_qty + order by group_num + ''' + + def requires(self) -> List[luigi.Task]: + groups = self.partition_patients() + return [ + EpicFactsLoadGroup(epic_fact_view=view, + enc_source_cd=self.enc_source_cd, + pat_id_lo=lo, + pat_id_hi=hi, + pat_group_qty=qty, + pat_group_num=num) + for view in self.fact_views + for (qty, num, lo, hi) in groups] + + def partition_patients(self) -> List[sqla.engine.RowProxy]: + # ISSUE: persist partition? + with self.connection('partition patients') as q: + groups = q.execute(self.pat_grp_q.format(i2b2_star=self.project.star_schema), + params=dict(group_qty=self.pat_group_qty)).fetchall() + q.log.info('groups: %s', groups) + return groups diff --git a/etl_tasks.py b/etl_tasks.py new file mode 100644 index 0000000..0090dc2 --- /dev/null +++ b/etl_tasks.py @@ -0,0 +1,899 @@ +'''etl_tasks -- Source-agnostic Luigi ETL Task support + +Note: This is source-agnostic but not target-agnositc; it has i2b2 + knowledge. + +''' + +from typing import Any, Callable, Dict, Iterator, List, Optional as Opt, Tuple, cast +from contextlib import contextmanager +from datetime import datetime +import csv +import logging + +from luigi.contrib.sqla import SQLAlchemyTarget +from sqlalchemy import text as sql_text, func, Table # type: ignore +from sqlalchemy.engine import Connection, Engine +from sqlalchemy.engine.result import ResultProxy +from sqlalchemy.engine.url import make_url +from sqlalchemy.exc import DatabaseError +from sqlalchemy.sql.expression import Select +import sqlalchemy as sqla +import luigi +from cx_Oracle import Error as OraError, _Error as Ora_Error + +from eventlog import EventLogger, LogState, JSONObject # ISSUE: use https://eliot.readthedocs.io/en/1.1.0/ instead? +from param_val import StrParam, IntParam, BoolParam +from script_lib import Script +from sql_syntax import Environment, Params, SQL +from sql_syntax import params_used, insert_append_table + +log = logging.getLogger(__name__) + + +class DBTarget(SQLAlchemyTarget): + '''Take advantage of engine caching logic from SQLAlchemyTarget, + but don't bother with target_table, update_id, etc. + + >>> t = DBTarget(connection_string='sqlite:///') + >>> t.engine.scalar('select 1 + 1') + 2 + ''' + def __init__(self, connection_string: str, + target_table: Opt[str]=None, update_id: Opt[str]=None, + echo: bool=False) -> None: + SQLAlchemyTarget.__init__( + self, connection_string, + target_table=target_table, + update_id=update_id, + echo=echo) + + def exists(self) -> bool: + raise NotImplementedError + + def touch(self) -> None: + raise NotImplementedError + + +class ETLAccount(luigi.Config): + '''Access to connect and run ETL. + + This account needs read access to source material, write access to + the destination i2b2 star schema, and working space for + intermediary tables, views, functions, and such. + ''' + account = StrParam(description='see client.cfg', + default='') + passkey = StrParam(description='see client.cfg', + default='', + significant=False) + ssh_tunnel = StrParam(description='see client.cfg', + default='', + significant=False) + echo = BoolParam(description='SQLAlchemy echo logging', + significant=False) + + +class LoggedConnection(object): + '''Wrap (parts of) the sqlalchemy Connection API with logging. + + If you have an EventLogger, `log`, you can wrap a connection + in a step that explains what the connection is for:: + + with log.step('crunching data') as step: + lc = LoggedConnection(conn, log, step) + + .. note:: A LoggedConnection wraps only the `execute` and `scalar` + methods from the sqlalchemy API. + + ''' + def __init__(self, conn: Connection, log: EventLogger, + step: LogState) -> None: + self._conn = conn + self.log = log + self.step = step + + def __repr__(self) -> str: + return '%s(%s, %s)' % (self.__class__.__name__, self._conn, self.log) + + def _log_args(self, event: str, operation: object, + params: Params) -> Tuple[str, JSONObject, JSONObject]: + msg = '%(event)s %(sql3)s' + ('\n%(params)s' if params else '') + argobj = dict(event=event, sql3=_peek(operation, lines=3), params=params) + extra = dict(statement=str(operation)) + return msg, argobj, extra + + def execute(self, operation: object, params: Opt[Params] = None) -> ResultProxy: + msg, argobj, extra = self._log_args('execute', operation, params or {}) + with self.log.step(msg, argobj, extra): + return self._conn.execute(operation, params or {}) + + def scalar(self, operation: object, params: Opt[Params] = None) -> Any: + msg, argobj, extra = self._log_args('scalar', str(operation), params or {}) + with self.log.step(msg, argobj, extra) as step: + result = self._conn.scalar(operation, params or {}) + step.extra.update(dict(result=result)) + return result + + +def _peek(thing: object, + lines: int=1, + max_len: int=120) -> str: + return '\n'.join(str(thing).split('\n')[:lines])[:180] + + +class DBAccessTask(luigi.Task): + """Manage DB account credentials and logging. + + Typical usage:: + + with self.connection(event='munching data') as conn: + howmany = conn.scalar('select count(*) from cookies') + yummy = conn.execute('select * from cookies') + """ + account = StrParam(default=ETLAccount().account) + ssh_tunnel = StrParam(default=ETLAccount().ssh_tunnel, + significant=False) + passkey = StrParam(default=ETLAccount().passkey, + significant=False) + echo = BoolParam(default=ETLAccount().echo, + significant=False) + max_idle = IntParam(description='Set to less than Oracle profile max idle time.', + default=60 * 20, + significant=False) + _log = logging.getLogger(__name__) # ISSUE: ambient. + + def output(self) -> luigi.Target: + return self._dbtarget() + + def _dbtarget(self) -> DBTarget: + return DBTarget(self._make_url(self.account), + target_table=None, update_id=self.task_id, + echo=self.echo) + + def _make_url(self, account: str) -> str: + url = make_url(account) + if 'oracle' in account.lower(): + url.query['pool_recycle'] = self.max_idle + # `twophase` interferes with direct path load somehow. + # To disable twophase, allow_twophase flag was commented out of opts dict in cx_Oracle.py + # This appears to be a change from cx_Oracle version 5 to 6. + url.query['allow_twophase'] = False + if self.passkey: + from os import environ # ISSUE: ambient + url.password = environ[self.passkey] + if self.ssh_tunnel and url.host: + host, port = self.ssh_tunnel.split(':', 1) + url.host = host + url.port = port + return str(url) + + def log_info(self) -> Dict[str, Any]: + '''Get info to log: luigi params, task_family, task_hash. + ''' + return dict(self.to_str_params(only_significant=True), + task_family=self.task_family, + task_hash=self.task_id[-luigi.task.TASK_ID_TRUNCATE_HASH:]) + + @contextmanager + def connection(self, event: str='connect') -> Iterator[LoggedConnection]: + conn = ConnectionProblem.tryConnect(self._dbtarget().engine) + log = EventLogger(self._log, self.log_info()) + with log.step('%(event)s: <%(account)s>', + dict(event=event, account=self.account)) as step: + yield LoggedConnection(conn, log, step) + + def _fix_password(self, environ: Dict[str, str], getpass: Callable[[str], str]) -> None: + '''for interactive use; e.g. in notebooks + ''' + if self.passkey not in environ: + environ[self.passkey] = getpass(self.passkey) + + +class SqlScriptTask(DBAccessTask): + '''Task to run a stylized SQL script. + + As seen in `script_lib`, a script may require (in the luigi sense) + other scripts and it is complete iff its last query says so. + + Running a script may be parameterized with bind params and/or + Oracle sqlplus style defined `&&variables`: + + >>> variables = dict(I2B2STAR='I2B2DEMODATA') + >>> txform = SqlScriptTask( + ... account='sqlite:///', passkey=None, + ... script=Script.migrate_fact_upload, + ... param_vars=variables) + + ISSUE: doctest dependencies? + >>> [task.script for task in txform.requires()] + ... #doctest: +ELLIPSIS + [] + + >>> txform.complete() + False + + ''' + script = cast(Script, luigi.EnumParameter(enum=Script)) + param_vars = cast(Environment, luigi.DictParameter(default={})) + _log = logging.getLogger('sql_scripts') # ISSUE: ambient. magic-string + + @property + def variables(self) -> Environment: + '''Defined variables for this task (or task family). + ''' + return self.param_vars + + @property + def vars_for_deps(self) -> Environment: + '''Defined variables to supply to dependencies. + ''' + return self.variables + + def requires(self) -> List[luigi.Task]: + '''Wrap each of `self.script.deps()` in a SqlScriptTask. + ''' + return [SqlScriptTask(script=s, + param_vars=self.vars_for_deps, + account=self.account, + passkey=self.passkey, + echo=self.echo) + for s in self.script.deps()] + + def log_info(self) -> Dict[str, Any]: + '''Include script, filename in self.log_info(). + ''' + return dict(DBAccessTask.log_info(self), + script=self.script.name, + filename=self.script.fname) + + def complete(self) -> bool: + '''Each script's last query tells whether it is complete. + + It should be a scalar query that returns non-zero for done + and either zero or an error for not done. + ''' + last_query = self.last_query() + params = params_used(self.complete_params(), last_query) + with self.connection(event=self.task_family + ' complete query: ' + self.script.name) as conn: + try: + result = conn.scalar(sql_text(last_query), params) + return bool(result) + except DatabaseError as exc: + conn.log.warning('%(event)s: %(exc)s', + dict(event='complete query error', exc=exc)) + return False + + def last_query(self) -> SQL: + """ + Note: In order to support run-only variables as in UploadTask, + we skip statements with unbound &&variables. + """ + return self.script.statements( + skip_unbound=True, + variables=self.variables)[-1] + + def complete_params(self) -> Dict[str, Any]: + '''Make `task_id` available to complete query as a bind param. + ''' + return dict(task_id=self.task_id) + + def run(self) -> None: + '''Run each statement in the script without any bind parameters. + ''' + self.run_bound() + + def run_bound(self, + script_params: Opt[Params]=None) -> None: + '''Run with a (default emtpy) set of parameters bound. + ''' + with self.connection(event='run script') as conn: + self.run_event(conn, script_params=script_params) + + def run_event(self, + conn: LoggedConnection, + run_vars: Opt[Environment]=None, + script_params: Opt[Params]=None) -> int: + '''Run script inside a LoggedConnection event. + + @param run_vars: variables to define for this run + @param script_params: parameters to bind for this run + @return: count of rows bulk-inserted + always 0 for this class, but see UploadTask + + To see how a script can ignore errors, see :mod:`script_lib`. + ''' + bulk_rows = 0 + ignore_error = False + run_params = dict(script_params or {}, task_id=self.task_id) + fname = self.script.fname + variables = dict(run_vars or {}, **self.variables) + each_statement = self.script.each_statement(variables=variables) + + for line, _comment, statement in each_statement: + try: + if self.is_bulk(statement): + bulk_rows = self.bulk_insert( + conn, fname, line, statement, run_params, + bulk_rows) + else: + ignore_error = self.execute_statement( + conn, fname, line, statement, run_params, + ignore_error) + except DatabaseError as exc: + db = self._dbtarget().engine + err = SqlScriptError(exc, self.script, line, + statement, str(db)) + if ignore_error: + conn.log.warning('%(event)s: %(error)s', + dict(event='ignore', error=err)) + else: + raise err from None + if bulk_rows > 0: + conn.step.msg_parts.append(' %(rowtotal)s total rows') + conn.step.argobj.update(dict(rowtotal=bulk_rows)) + + return bulk_rows + + def execute_statement(self, conn: LoggedConnection, fname: str, line: int, + statement: SQL, run_params: Params, + ignore_error: bool) -> bool: + '''Log and execute one statement. + ''' + sqlerror = Script.sqlerror(statement) + if sqlerror is not None: + return sqlerror + params = params_used(run_params, statement) + self.set_status_message( + '%s:%s:\n%s\n%s' % (fname, line, statement, params)) + conn.execute(statement, params) + return ignore_error + + def is_bulk(self, statement: SQL) -> bool: + '''always False for this class, but see UploadTask + ''' + return False + + def bulk_insert(self, conn: LoggedConnection, fname: str, line: int, + statement: SQL, run_params: Params, + bulk_rows: int) -> int: + raise NotImplementedError( + 'overriding is_bulk() requires overriding bulk_insert()') + + +def log_plan(lc: LoggedConnection, event: str, params: Dict[str, Any], + query: Opt[Select]=None, sql: Opt[str]=None) -> None: + if query is not None: + sql = str(query.compile(bind=lc._conn)) + if sql is None: + return + plan = explain_plan(lc, sql) + param_msg = ', '.join('%%(%s)s' % k for k in params.keys()) + lc.log.info('%(event)s [' + param_msg + ']\n' + 'query: %(query_peek)s plan:\n' + '%(plan)s', + dict(params, event=event, query_peek=_peek(sql), + plan='\n'.join(plan))) + + +def explain_plan(work: LoggedConnection, statement: SQL) -> List[str]: + work.execute('explain plan for ' + statement) + # ref 19 Using EXPLAIN PLAN + # Oracle 10g Database Performance Tuning Guide + # https://docs.oracle.com/cd/B19306_01/server.102/b14211/ex_plan.htm + plan = work.execute( + 'SELECT PLAN_TABLE_OUTPUT line FROM TABLE(DBMS_XPLAN.DISPLAY())') + return [row.line for row in plan] # type: ignore # sqla + + +def maybe_ora_err(exc: Exception) -> Opt[Ora_Error]: + if isinstance(exc, DatabaseError): + if isinstance(exc.orig, OraError): + return cast(Ora_Error, exc.orig.args[0]) + return None + + +class SqlScriptError(IOError): + '''Include script file, line number in diagnostics + ''' + def __init__(self, exc: Exception, script: Script, line: int, statement: SQL, + conn_label: str) -> None: + fname = script.name + message = '%s <%s>\n%s:%s:\n' + args = [exc, conn_label, fname, line] + ora_ex = maybe_ora_err(exc) + if ora_ex: + offset = ora_ex.offset + message += '%s%s' + args[0] = ora_ex.message + args += [_pick_lines(statement[:offset], -3, None), + _pick_lines(statement[offset:], None, 3)] + else: + message += '%s' + args += [statement] + + self.message = message + self.args = tuple(args) + + def __str__(self) -> str: + return self.message % self.args + + +def _pick_lines(s: str, lo: Opt[int], hi: Opt[int]) -> str: + return '\n'.join(s.split('\n')[lo:hi]) + + +class TimeStampParameter(luigi.Parameter): + '''A datetime interchanged as milliseconds since the epoch. + ''' + + def parse(self, s: str) -> datetime: + ms = int(s) + return datetime.fromtimestamp(ms / 1000.0) + + def serialize(self, dt: datetime) -> str: + epoch = datetime.utcfromtimestamp(0) + ms = (dt - epoch).total_seconds() * 1000 + return str(int(ms)) + + +class SourceTask(luigi.Task): + @property + def source_cd(self) -> str: + raise NotImplementedError + + @property + def download_date(self) -> datetime: + raise NotImplementedError + + +class I2B2Task(object): + @property + def project(self) -> 'I2B2ProjectCreate': + return I2B2ProjectCreate() + + +class UploadTask(I2B2Task, SqlScriptTask): + '''Run a script with an associated `upload_status` record. + ''' + @property + def source(self) -> SourceTask: + raise NotImplementedError('subclass must implement') + + @property + def transform_name(self) -> str: + return self.task_id + + def complete_params(self) -> Dict[str, Any]: + return dict(task_id=self.task_id, + download_date=self.source.download_date) + + def output(self) -> luigi.Target: + return self._upload_target() + + def _upload_target(self) -> 'UploadTarget': + return UploadTarget(self._make_url(self.account), + self.project.upload_table, + self.transform_name, self.source, + echo=self.echo) + + def requires(self) -> List[luigi.Task]: + return [self.project, self.source] + SqlScriptTask.requires(self) + + def complete(self) -> bool: + # Belt and suspenders + return (self.output().exists() and + SqlScriptTask.complete(self)) + + @property + def label(self) -> str: + return self.script.title + + def run(self) -> None: + upload = self._upload_target() + with upload.job(self, + label=self.label, + user_id=make_url(self.account).username) as conn_id_r: + conn, upload_id, result = conn_id_r + bulk_rows = SqlScriptTask.run_event( + self, conn, + run_vars=dict(upload_id=str(upload_id)), + script_params=dict(self.script_params(), upload_id=upload_id)) + result[upload.table.c.loaded_record.name] = bulk_rows + + def script_params(self) -> Params: + return dict(download_date=self.source.download_date, + project_id=self.project.project_id) + + def is_bulk(self, statement: SQL) -> bool: + return insert_append_table(statement) is not None + + def bulk_insert(self, conn: LoggedConnection, fname: str, line: int, + statement: SQL, run_params: Params, + bulk_rows: int) -> int: + with conn.log.step( + '%(filename)s:%(lineno)s: %(event)s', + dict(event='bulk_insert', + filename=fname, lineno=line)) as step: + plan = '\n'.join(explain_plan(conn, statement)) + conn.log.info('%(filename)s:%(lineno)s: %(event)s:\n%(plan)s', + dict(filename=fname, lineno=line, event='plan', + plan=plan)) + + params = params_used(run_params, statement) + self.set_status_message( + '%s:%s:\n%s\n%s' % (fname, line, statement, params)) + last_result = conn.execute(statement, params) + step.msg_parts.append(' %(rowcount)d rows') + step.argobj.update(dict(rowcount=last_result.rowcount)) + bulk_rows += last_result.rowcount + + return bulk_rows + + +class UploadTarget(DBTarget): + def __init__(self, connection_string: str, + table: sqla.Table, transform_name: str, source: SourceTask, + echo: bool=False) -> None: + DBTarget.__init__(self, connection_string, + echo=echo) + self.table = table + self.source = source + self.transform_name = transform_name + self.upload_id = None # type: Opt[int] + + def __repr__(self) -> str: + return '%s(transform_name=%s)' % ( + self.__class__.__name__, self.transform_name) + + def exists(self) -> bool: + conn = ConnectionProblem.tryConnect(self.engine) + with conn.begin(): + up_t = self.table + upload_id = conn.scalar( + sqla.select([sqla.func.max(up_t.c.upload_id)]) + .select_from(up_t) + .where(sqla.and_(up_t.c.transform_name == self.transform_name, + up_t.c.load_status == 'OK'))) + return upload_id is not None + + @contextmanager + def job(self, task: DBAccessTask, + label: Opt[str] = None, user_id: Opt[str] = None, + upload_id: Opt[int] = None) -> Iterator[ + Tuple[LoggedConnection, int, Params]]: + event = 'upload job' + with task.connection(event=event) as conn: + up_t = self.table + if upload_id is None: + if user_id is None: + raise TypeError('must supply user_id for new record') + if label is None: + raise TypeError('must supply label for new record') + upload_id = self.insert(conn, label, user_id) + else: + [label, user_id] = conn.execute( + sqla.select([up_t.c.upload_label, up_t.c.user_id]) + .where(up_t.c.upload_id == upload_id)).fetchone() + + msg = ' %(upload_id)s for %(label)s' + info = dict(label=label, upload_id=upload_id) + conn.step.msg_parts.append(msg) + conn.step.argobj.update(info) + conn.log.info(msg, info) # Go ahead and log the upload_id early. + + result = {} # type: Params + yield conn, upload_id, result + conn.execute(up_t.update() + .where(up_t.c.upload_id == upload_id) + .values(load_status='OK', end_date=func.now(), + **result)) + + def insert(self, conn: LoggedConnection, label: str, user_id: str) -> int: + ''' + :param label: a label for related facts for audit purposes + :param user_id: an indication of who uploaded the related facts + ''' + up_t = self.table + next_q = sql_text( + '''select {i2b2}.sq_uploadstatus_uploadid.nextval + from dual'''.format(i2b2=self.table.schema)) + upload_id = conn.scalar(next_q) # type: int + + conn.execute(up_t.insert() + .values(upload_id=upload_id, + upload_label=label, + user_id=user_id, + source_cd=self.source.source_cd, + load_date=sqla.func.now(), + transform_name=self.transform_name)) + self.upload_id = upload_id + return upload_id + + +class I2B2ProjectCreate(DBAccessTask): + star_schema = StrParam(description='see client.cfg') + project_id = StrParam(description='see client.cfg') + _meta = None # type: Opt[sqla.MetaData] + _upload_table = None # type: Opt[sqla.Table] + + Column, ty = sqla.Column, sqla.types + upload_status_columns = [ + Column('upload_id', ty.Numeric(38, 0, asdecimal=False), primary_key=True), + Column('upload_label', ty.String(500), nullable=False), + Column('user_id', ty.String(100), nullable=False), + Column('source_cd', ty.String(50), nullable=False), + Column('no_of_record', ty.Numeric(asdecimal=False)), + Column('loaded_record', ty.Numeric(asdecimal=False)), + Column('deleted_record', ty.Numeric(asdecimal=False)), + Column('load_date', ty.DateTime, nullable=False), + Column('end_date', ty.DateTime), + Column('load_status', ty.String(100)), + Column('message', ty.Text), + Column('input_file_name', ty.Text), + Column('log_file_name', ty.Text), + Column('transform_name', ty.String(500)), + ] + + def output(self) -> 'SchemaTarget': + return SchemaTarget(self._make_url(self.account), + schema_name=self.star_schema, + table_eg='patient_dimension', + echo=self.echo) + + def run(self) -> None: + raise NotImplementedError('see heron_create.create_deid_datamart etc.') + + @property + def metadata(self) -> sqla.MetaData: + if self._meta: + return self._meta + self._meta = meta = sqla.MetaData(schema=self.star_schema) + return meta + + def table_details(self, lc: LoggedConnection, tables: List[str]) -> sqla.MetaData: + i2b2_meta = sqla.MetaData(schema=self.star_schema) + i2b2_meta.reflect(only=tables, schema=self.star_schema, + bind=self._dbtarget().engine) + return i2b2_meta + + @property + def upload_table(self) -> sqla.Table: + if self._upload_table is not None: + return self._upload_table + t = sqla.Table( + 'upload_status', self.metadata, + *self.upload_status_columns, + schema=self.star_schema) + self._upload_table = t + return t + + +class SchemaTarget(DBTarget): + def __init__(self, connection_string: str, schema_name: str, table_eg: str, + echo: bool=False) -> None: + DBTarget.__init__(self, connection_string, echo=echo) + self.schema_name = schema_name + self.table_eg = table_eg + + def exists(self) -> bool: + table = Table(self.table_eg, sqla.MetaData(), schema=self.schema_name) + return table.exists(bind=self.engine) # type: ignore + + +class ConnectionProblem(DatabaseError): + '''Provide hints about ssh tunnels. + ''' + # connection closed, no listener + tunnel_hint_codes = [12537, 12541] + + @classmethod + def tryConnect(cls, engine: Engine) -> Connection: + try: + return engine.connect() + except DatabaseError as exc: + raise ConnectionProblem.refine(exc, str(engine)) from None + + @classmethod + def refine(cls, exc: Exception, conn_label: str) -> Exception: + '''Recognize known connection problems. + + :returns: customized exception for known + problem else exc + ''' + ora_ex = maybe_ora_err(exc) + + if ora_ex: + return cls(exc, ora_ex, conn_label) + return exc + + def __init__(self, exc: DatabaseError, ora_ex: Ora_Error, conn_label: str) -> None: + DatabaseError.__init__( + self, + exc.statement, exc.params, + exc.connection_invalidated) + message = '%s <%s>' + args = [ora_ex, conn_label] + + if exc.statement and ora_ex.offset: + stmt_rest = exc.statement[ + ora_ex.offset:ora_ex.offset + 120] + message += '\nat: %s' + args += [stmt_rest] + local_conn_prob = ( + ora_ex.code in self.tunnel_hint_codes and + 'localhost' in conn_label) + if local_conn_prob: + message += '\nhint: ssh tunnel down?' + message += '\nin: %s' + args += [ora_ex.context] + self.message = message + self.args = tuple(args) + + def __str__(self) -> str: + return self.message % self.args + + +class ReportTask(DBAccessTask): + @property + def script(self) -> Script: + raise NotImplementedError('subclass must implement') + + @property + def report_name(self) -> str: + raise NotImplementedError('subclass must implement') + + def complete(self) -> bool: + '''Double-check requirements as well as output. + ''' + deps = luigi.task.flatten(self.requires()) # type: List[luigi.Task] + return (self.output().exists() and + all(t.complete() for t in deps)) + + def _csvout(self) -> 'CSVTarget': + return CSVTarget(path=self.report_name + '.csv') + + def output(self) -> luigi.Target: + return self._csvout() + + def run(self) -> None: + with self.connection('report') as conn: + query = sql_text( + 'select * from {object}'.format(object=self.report_name)) + result = conn.execute(query) + cols = result.keys() + rows = result.fetchall() + self._csvout().export(cols, rows) + + +class CSVTarget(luigi.local_target.LocalTarget): + def export(self, cols: List[str], data: List) -> None: + with self.open('wb') as stream: + dest = csv.writer(stream) + dest.writerow(cols) + dest.writerows(data) + + @contextmanager + def dictreader(self, + lowercase_fieldnames: bool=False, + delimiter: str=',') -> Iterator[csv.DictReader]: + '''DictReader contextmanager + + @param lowercase_fieldnames: sqlalchemy uses lower-case bind + parameter names, but SCILHS CSV file headers use the + actual uppercase column names. So we got: + + CompileError: The 'oracle' dialect with current + database version settings does not support empty + inserts. + + + ''' + with self.open('rb') as stream: + dr = csv.DictReader(stream, delimiter=delimiter) + if lowercase_fieldnames: + # This is a bit of a kludge, but it works... + dr.fieldnames = [n.lower() for n in dr.fieldnames] + yield dr + + +class AdHoc(DBAccessTask): + sql = StrParam() + name = StrParam() + + def _csvout(self) -> CSVTarget: + return CSVTarget(path=self.name + '.csv') + + def output(self) -> luigi.Target: + return self._csvout() + + def run(self) -> None: + with self.connection() as work: + result = work.execute(self.sql) + cols = result.keys() + rows = result.fetchall() + self._csvout().export(cols, rows) + + +class KillSessions(DBAccessTask): + reason = StrParam(default='*no reason given*') + sql = ''' + begin + sys.kill_own_sessions(:reason); + end; + ''' + + def complete(self) -> bool: + return False + + def run(self) -> None: + with self.connection('kill own sessions') as work: + work.execute(self.sql, params=dict(reason=self.reason)) + + +class AlterStarNoLogging(DBAccessTask): + sql = ''' + alter table TABLE nologging + ''' + tables = ['patient_mapping', + 'encounter_mapping', + 'patient_dimension', + 'visit_dimension', + 'observation_fact'] + + def complete(self) -> bool: + return False + + def run(self) -> None: + with self.connection() as work: + for table in self.tables: + work.execute(self.sql.replace('TABLE', table)) + + +class MigrateUpload(SqlScriptTask, I2B2Task): + upload_id = IntParam() + workspace_star = StrParam() + parallel_degree = IntParam(default=24, + significant=False) + + script = Script.migrate_fact_upload + + @property + def variables(self) -> Environment: + return dict(I2B2STAR=self.project.star_schema, + workspace_star=self.workspace_star, + parallel_degree=str(self.parallel_degree), + upload_id=str(self.upload_id)) + + +class MigratePendingUploads(DBAccessTask, I2B2Task, luigi.WrapperTask): + workspace_star = StrParam() + + find_pending = """ + select upload_id from %(WORKSPACE)s.upload_status + where load_status in ('OK', 'OK_work') and upload_id not in ( + select upload_id from %(I2B2STAR)s.upload_status + where load_status='OK' ) + """ + + def requires(self) -> List[luigi.Task]: + find_pending = self.find_pending % dict( + WORKSPACE=self.workspace_star, + I2B2STAR=self.project.star_schema) + + deps = [] # type: List[luigi.Task] + with self.connection('pending uploads') as lc: + pending = [row.upload_id for row in + lc.execute(find_pending).fetchall()] + + workmeta = sqla.MetaData() + for upload_id in pending: + table = Table('observation_fact_%d' % upload_id, workmeta, + schema=self.workspace_star) + if table.exists(bind=lc._conn): + deps.append( + MigrateUpload(upload_id=upload_id, + workspace_star=self.workspace_star)) + else: + log.warn('no such table to migrate: %s', table) + return deps diff --git a/eventlog.py b/eventlog.py new file mode 100644 index 0000000..1de5799 --- /dev/null +++ b/eventlog.py @@ -0,0 +1,152 @@ +'''eventlog -- structured logging support + +EventLogger for nested events +============================= + +Suppose we want to log some events which naturally nest. + +First, add a handler to a logger: + +>>> import logging, sys +>>> log1 = logging.getLogger('log1') +>>> detail = logging.StreamHandler(sys.stdout) +>>> log1.addHandler(detail) +>>> log1.setLevel(logging.INFO) +>>> detail.setFormatter(logging.Formatter( +... fmt='%(levelname)s %(elapsed)s %(do)s %(message)s')) + +>>> io = MockIO() +>>> event0 = EventLogger(log1, dict(customer='Jones', invoice=123), io.clock) + + +>>> with event0.step('Build %(product)s', dict(product='house')): +... with event0.step('lay foundation %(depth)d ft deep', +... dict(depth=20)) as info: +... info.msg_parts.append(' at %(temp)d degrees') +... info.argobj['temp'] = 65 +... with event0.step('frame %(stories)d story house', +... dict(stories=2)) as info: +... pass +... start, elapsed, _ms = event0.elapsed() +... eta = event0.eta(pct=25) +... # doctest: +ELLIPSIS +INFO ('... 12:30:01', None, None) begin 0:00:00 [1] Build house... +INFO ('... 12:30:03', None, None) begin 0:00:02 [1, 2] lay foundation 20 ft deep... +INFO ('... 12:30:03', '0:00:03', 3000000) end 0:00:03 [1, 2] lay foundation 20 ft deep at 65 degrees. +INFO ('... 12:30:10', None, None) begin 0:00:09 [1, 3] frame 2 story house... +INFO ('... 12:30:10', '0:00:05', 5000000) end 0:00:05 [1, 3] frame 2 story house. +INFO ('... 12:30:01', '0:00:35', 35000000) end 0:00:35 [1] Build house. + +>>> eta +datetime.datetime(2000, 1, 1, 12, 31, 49) + +''' + +from contextlib import contextmanager +from datetime import datetime +from typing import ( + Any, Callable, Dict, Iterator, List, MutableMapping, + NamedTuple, Optional as Opt, TextIO, Tuple +) +import logging + + +KWArgs = MutableMapping[str, Any] +JSONObject = Dict[str, Any] + +LogState = NamedTuple('LogState', [ + ('msg_parts', List[str]), + ('argobj', JSONObject), + ('extra', JSONObject)]) + + +class EventLogger(logging.LoggerAdapter): + def __init__(self, logger: logging.Logger, event: JSONObject, + clock: Opt[Callable[[], datetime]]=None) -> None: + logging.LoggerAdapter.__init__(self, logger, extra={}) +# self.name = logger.name + if clock is None: + clock = datetime.now # ISSUE: ambient + self.event = event + self._clock = clock + self._seq = 0 + self._step = [] # type: List[Tuple[int, datetime]] + List # let flake8 know we're using it + + def __repr__(self) -> str: + return '%s(%s, %s)' % (self.__class__.__name__, self.name, self.event) + + def process(self, msg: str, kwargs: KWArgs) -> Tuple[str, KWArgs]: + extra = dict(kwargs.get('extra', {}), + context=self.event) + return msg, dict(kwargs, extra=extra) + + def elapsed(self, then: Opt[datetime]=None) -> Tuple[str, str, int]: + start = then or self._step[-1][1] + elapsed = self._clock() - start + ms = int(elapsed.total_seconds() * 1000000) + return (str(start), str(elapsed), ms) + + def eta(self, pct: float) -> datetime: + t0 = self._step[0][1] + elapsed = self._clock() - t0 + return t0 + elapsed * (1 / (pct / 100)) + + @contextmanager + def step(self, msg: str, argobj: Dict[str, object], + extra: Opt[Dict[str, object]]=None) -> Iterator[LogState]: + checkpoint = self._clock() + self._seq += 1 + self._step.append((self._seq, checkpoint)) + extra = extra or {} + fmt_step = '%(t_step)s %(step)s ' + step_ixs = [ix for (ix, _t) in self._step] + t_step = str(checkpoint - self._step[0][1]) + self.info(fmt_step + msg + '...', + dict(argobj, step=step_ixs, t_step=t_step), + extra=dict(extra, do='begin', + elapsed=(str(checkpoint), None, None))) + msgparts = [msg] + outcome = logging.INFO + try: + yield LogState(msgparts, argobj, extra) + except: + outcome = logging.ERROR + raise + finally: + elapsed = self.elapsed(then=checkpoint) + self.log(outcome, ''.join([fmt_step] + msgparts) + '.', + dict(argobj, step=step_ixs, t_step=elapsed[1]), + extra=dict(extra, do='end', + elapsed=elapsed)) + self._step.pop() + + +class TextFilter(logging.Filter): + def __init__(self, skips: List[str]) -> None: + self.skips = skips + + def filter(self, record: logging.LogRecord) -> bool: + for skip in self.skips: + if record.getMessage().startswith(skip): + return False + return True + + +class TextHandler(logging.StreamHandler): + def __init__(self, stream: TextIO, skips: List[str]=[]) -> None: + logging.StreamHandler.__init__(self, stream) + self.addFilter(TextFilter(skips)) + + +class MockIO(object): + def __init__(self, + now: datetime=datetime(2000, 1, 1, 12, 30, 0)) -> None: + self._now = now + self._delta = 1 + + def clock(self) -> datetime: + from datetime import timedelta + self._now += timedelta(seconds=self._delta) + self._delta += 1 + return self._now diff --git a/i2p_tasks.py b/i2p_tasks.py new file mode 100644 index 0000000..553bece --- /dev/null +++ b/i2p_tasks.py @@ -0,0 +1,11 @@ +from etl_tasks import SqlScriptTask +from script_lib import Script +from sql_syntax import Environment + + +class PCORNetInit(SqlScriptTask): + script = Script.PCORNetInit + + @property + def variables(self) -> Environment: + return dict(datamart_id='todo', datamart_name='todo2', i2b2_data_schema='todo3') diff --git a/logging.cfg b/logging.cfg new file mode 100644 index 0000000..efc350e --- /dev/null +++ b/logging.cfg @@ -0,0 +1,83 @@ +# see section 15.8.3. Configuration file format in +# http://docs.python.org/2/library/logging.config.html +# +# also: +# https://github.com/pysysops/docker-luigi-taskrunner/blob/master/etc/luigi/logging.cfg +# https://pypi.python.org/pypi/python-json-logger +# https://stedolan.github.io/jq/manual/v1.5/ +# +# Logging to JSON lets us do fun stuff such as: +# +# $ < log/grouse-detail.json jq --compact-output -C \ +# 'select(.args|objects|.event == "inserted chunk") | +# [.asctime, .process, .args.filename, .args.lineno, .args.into, +# {elapsed: (.elapsed[1] | split("."))[0], +# rowcount: .args.rowcount, +# krowpersec: (.args.rowcount / .elapsed[2] * 1000 * 60 + 0.5 | floor)}]' +# ["2017-03-16 21:20:24",10770,"cms_patient_dimension.sql",9,"\"DCONNOLLY\".patient_dimension", +# {"elapsed":"0:01:39","rowcount":419835,"krowpersec":252}] +# ["2017-03-16 21:22:12",10770,"cms_patient_dimension.sql",9,"\"DCONNOLLY\".patient_dimension", +# {"elapsed":"0:01:47","rowcount":419835,"krowpersec":234}] + + +[loggers] +keys=root,luigi_debug + +[logger_root] +level=INFO +#handlers=console,detail +handlers=console + +[logger_luigi_debug] +level=DEBUG +handlers=luigi_debug +qualname=luigi-interface + +[handlers] +#keys=console,detail,luigi_debug +keys=console,luigi_debug + +[handler_console] +level=INFO +# class=StreamHandler +class=eventlog.TextHandler +args=(sys.stderr, ['execute', 'execute commit', 'complete query', 'find chunks']) +# args=(3, 0.1) +formatter=timed + +[handler_detail] +level=INFO +class=FileHandler +# use detail_log_dir rather than dir to facilitate stream editing +detail_log_dir=log +detail_log_file=%(detail_log_dir)s/grouse-detail.json +# append to log file +#args=('%(detail_log_file)s', 'a', None, True) +# overwrite log file +args=('%(detail_log_file)s', 'w') +formatter=json + +[handler_luigi_debug] +level=DEBUG +class=FileHandler +luigi_debug_log_dir=log +luigi_debug_log_file=%(luigi_debug_log_dir)s/grouse-luigi-debug.json +# append to log file +# args=('%(luigi_debug_log_file)s', 'a') +# overwrite log file +args=('%(luigi_debug_log_file)s', 'w') +formatter=json + +[formatters] +keys=timed, json + +[formatter_timed] +class=logging.Formatter +# %(name)s? +format=%(asctime)s %(process)s %(levelname)s: %(message)s +#datefmt=%02H:%02M:%02S + +[formatter_json] +class = pythonjsonlogger.jsonlogger.JsonFormatter +format=%(asctime)s %(process)s %(name) %(levelname): %(message)s %(args)s +#datefmt=%Y-%m-%02d %02H:%02M:%02S diff --git a/param_val.py b/param_val.py new file mode 100644 index 0000000..1b4c4aa --- /dev/null +++ b/param_val.py @@ -0,0 +1,37 @@ +# TODO: module doc: luigi's abuse of class attributes vs. mypy static typing +from datetime import datetime +from typing import Any, Callable, TypeVar, cast + +import luigi + + +_T = TypeVar('_T') +_U = TypeVar('_U') + + +def _valueOf(example: _T, cls: Callable[..., _U]) -> Callable[..., _T]: + + def getValue(*args: Any, **kwargs: Any) -> _T: + return cast(_T, cls(*args, **kwargs)) + return getValue + + +class TimeStampParameter(luigi.Parameter): + '''A datetime interchanged as milliseconds since the epoch. + ''' + + def parse(self, s: str) -> datetime: + ms = int(s) + return datetime.fromtimestamp(ms / 1000.0) + + def serialize(self, dt: datetime) -> str: + epoch = datetime.utcfromtimestamp(0) + ms = (dt - epoch).total_seconds() * 1000 + return str(int(ms)) + + +StrParam = _valueOf('s', luigi.Parameter) +IntParam = _valueOf(0, luigi.IntParameter) +BoolParam = _valueOf(True, luigi.BoolParameter) +DictParam = _valueOf({'k': 'v'}, luigi.DictParameter) +TimeStampParam = _valueOf(datetime(2001, 1, 1, 0, 0, 0), TimeStampParameter) diff --git a/pythonjsonlogger/__init__.py b/pythonjsonlogger/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pythonjsonlogger/jsonlogger.py b/pythonjsonlogger/jsonlogger.py new file mode 100644 index 0000000..4f96331 --- /dev/null +++ b/pythonjsonlogger/jsonlogger.py @@ -0,0 +1,141 @@ +''' +This library is provided to allow standard python logging +to output log data as JSON formatted strings +''' +import logging +import json +import re +import datetime +import traceback + +from inspect import istraceback + +#Support order in python 2.7 and 3 +try: + from collections import OrderedDict +except ImportError: + pass + +# skip natural LogRecord attributes +# http://docs.python.org/library/logging.html#logrecord-attributes +RESERVED_ATTRS = ( + 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename', + 'funcName', 'levelname', 'levelno', 'lineno', 'module', + 'msecs', 'message', 'msg', 'name', 'pathname', 'process', + 'processName', 'relativeCreated', 'stack_info', 'thread', 'threadName') + +RESERVED_ATTR_HASH = dict(zip(RESERVED_ATTRS, RESERVED_ATTRS)) + + +def merge_record_extra(record, target, reserved=RESERVED_ATTR_HASH): + """ + Merges extra attributes from LogRecord object into target dictionary + + :param record: logging.LogRecord + :param target: dict to update + :param reserved: dict or list with reserved keys to skip + """ + for key, value in record.__dict__.items(): + #this allows to have numeric keys + if (key not in reserved + and not (hasattr(key, "startswith") + and key.startswith('_'))): + target[key] = value + return target + + +class JsonFormatter(logging.Formatter): + """ + A custom formatter to format logging records as json strings. + extra values will be formatted as str() if nor supported by + json default encoder + """ + + def __init__(self, *args, **kwargs): + """ + :param json_default: a function for encoding non-standard objects + as outlined in http://docs.python.org/2/library/json.html + :param json_encoder: optional custom encoder + :param json_serializer: a :meth:`json.dumps`-compatible callable + that will be used to serialize the log record. + :param prefix: an optional string prefix added at the beginning of + the formatted string + """ + self.json_default = kwargs.pop("json_default", None) + self.json_encoder = kwargs.pop("json_encoder", None) + self.json_serializer = kwargs.pop("json_serializer", json.dumps) + self.prefix = kwargs.pop("prefix", "") + #super(JsonFormatter, self).__init__(*args, **kwargs) + logging.Formatter.__init__(self, *args, **kwargs) + if not self.json_encoder and not self.json_default: + def _default_json_handler(obj): + '''Prints dates in ISO format''' + if isinstance(obj, (datetime.date, datetime.time)): + return obj.isoformat() + elif istraceback(obj): + tb = ''.join(traceback.format_tb(obj)) + return tb.strip() + elif isinstance(obj, Exception): + return "Exception: %s" % str(obj) + return str(obj) + self.json_default = _default_json_handler + self._required_fields = self.parse() + self._skip_fields = dict(zip(self._required_fields, + self._required_fields)) + self._skip_fields.update(RESERVED_ATTR_HASH) + + def parse(self): + """Parses format string looking for substitutions""" + standard_formatters = re.compile(r'\((.+?)\)', re.IGNORECASE) + return standard_formatters.findall(self._fmt) + + def add_fields(self, log_record, record, message_dict): + """ + Override this method to implement custom logic for adding fields. + """ + for field in self._required_fields: + log_record[field] = record.__dict__.get(field) + log_record.update(message_dict) + merge_record_extra(record, log_record, reserved=self._skip_fields) + + def process_log_record(self, log_record): + """ + Override this method to implement custom logic + on the possibly ordered dictionary. + """ + return log_record + + def jsonify_log_record(self, log_record): + """Returns a json string of the log record.""" + return self.json_serializer(log_record, + default=self.json_default, + cls=self.json_encoder) + + def format(self, record): + """Formats a log record and serializes to json""" + message_dict = {} + if isinstance(record.msg, dict): + message_dict = record.msg + record.message = None + else: + record.message = record.getMessage() + # only format time if needed + if "asctime" in self._required_fields: + record.asctime = self.formatTime(record, self.datefmt) + + # Display formatted exception, but allow overriding it in the + # user-supplied dict. + if record.exc_info and not message_dict.get('exc_info'): + message_dict['exc_info'] = self.formatException(record.exc_info) + if not message_dict.get('exc_info') and record.exc_text: + message_dict['exc_info'] = record.exc_text + + try: + log_record = OrderedDict() + except NameError: + log_record = {} + + self.add_fields(log_record, record, message_dict) + log_record = self.process_log_record(log_record) + + return "%s%s" % (self.prefix, self.jsonify_log_record(log_record)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4dd7554 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,55 @@ +# This is a python packaging requirements file. +# https://packaging.python.org/tutorials/installing-packages/#requirements-files + +# Usage (development): +# +# $ virtualenv --python python3.5 ~/pyenv/grouse3 +# $ . ~/pyenv/grouse3/bin/activate +# (grouse3)$ pip install -r requirements.txt +# ... +# Successfully built luigi tornado +# Installing collected packages: ... +# ... tornado-4.4.2 + + +# Production usage is more like: +# +# $ docker run --rm -ti --entrypoint=/bin/bash stockport/luigi-taskrunner +# app@8ee96e02e548:/$ . /luigi/.pyenv/bin/activate +# (.pyenv) app@8ee96e02e548:/$ python --version +# Python 3.5.2 +# (.pyenv) app@8ee96e02e548:/$ pip freeze +# alembic==0.8.9 + +# cx_Oracle isn't pure-python nor open source, so installation is +# failure-prone. Fortunately, the stockport/luigi-taskrunner Docker +# image provides it: + +#Changes from 5 to 6.0.3 have created a twophase incompatibility between +#SQLAlchemy and cx-Oracle. +cx-Oracle==6.0.3 + +# Cython==0.25.1 +# docutils==0.12 +# lockfile==0.12.2 +luigi==2.7.1 +# Mako==1.0.6 +# MarkupSafe==0.23 +# numpy==1.11.2 +pandas==0.19.1 +# psycopg2==2.6.2 +# pymssql==2.1.3 +# PyPDF2==1.26.0 +# python-daemon==2.1.2 +# python-dateutil==2.6.0 +# python-editor==1.0.3 +# pytz==2016.10 +# requests==2.12.3 +# six==1.10.0 +SQLAlchemy==1.1.4 +#tornado==4.4.2 +#XlsxWriter==0.9.4 + +# let's "vendor" it instead, since +# we have no Internet access from our build jobs. +# python-json-logger==0.1.7 diff --git a/script_lib.py b/script_lib.py new file mode 100644 index 0000000..c13e22c --- /dev/null +++ b/script_lib.py @@ -0,0 +1,313 @@ +r'''script_lib -- library of SQL scripts + +Scripts are pkg_resources, i.e. design-time constants. + +Each script should have a title, taken from the first line:: + + >>> Script.migrate_fact_upload.title + 'append data from a workspace table.' + + >>> text = Script.migrate_fact_upload.value + >>> lines = text.split('\n') + >>> print(lines[0]) + ... #doctest: +NORMALIZE_WHITESPACE + /** migrate_fact_upload - append data from a workspace table. + +We can separate the script into statements:: + + >>> statements = Script.epic_flowsheets_transform.statements() + >>> print(next(s for s in statements if 'insert' in s)) + ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) + with test_key as ( ... + +A bit of sqlplus syntax is supported for ignoring errors in just part +of a script: + + >>> Script.sqlerror('whenever sqlerror exit') + False + >>> Script.sqlerror('whenever sqlerror continue') + True + >>> Script.sqlerror('select 1 + 1 from dual') is None + True + +Dependencies between scripts are declared as follows:: + + >>> print(next(decl for decl in statements if "'dep'" in decl)) + ... #doctest: +ELLIPSIS + select test_name from etl_tests where 'dep' = 'etl_tests_init.sql' + + >>> Script.epic_flowsheets_transform.deps() + ... #doctest: +ELLIPSIS + [] + +We statically detect relevant effects; i.e. tables and views created:: + + >>> Script.epic_flowsheets_transform.created_objects() + ... #doctest: +ELLIPSIS + [view etl_test_domain_flowsheets, view flo_meas_type, view flowsheet_day, ... + +as well as tables inserted into:: + + >>> variables={I2B2STAR: 'I2B2DEMODATA', + ... CMS_RIF: 'CMS_DEID', 'upload_id': '20', 'chunk_qty': 20, + ... 'cms_source_cd': "'ccwdata.org'", 'source_table': 'T'} + >>> Script.epic_flowsheets_transform.inserted_tables(variables) + ['etl_test_values', 'approved_deid_flowsheets', 'etl_test_values'] + +The last statement should be a scalar query that returns non-zero to +signal that the script is complete: + + >>> print(statements[-1]) + ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + select 1 up_to_date + from epic_flowsheets_txform_sql where design_digest = &&design_digest + +The completion test may depend on a digest of the script and its dependencies: + + >>> design_digest = Script.epic_flowsheets_transform.digest() + >>> last = Script.epic_flowsheets_transform.statements(variables)[-1].strip() + >>> print(last) + ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + select 1 up_to_date + from epic_flowsheets_txform_sql where design_digest = ... + +ISSUE : Python hashes are senstive to the machine running the test? + +Some scripts use variables that are not known until a task is run; for +example, `&&upload_id` is used in names of objects such as tables and +partitions; these scripts must not refer to such variables in their +completion query: + + >>> del variables['upload_id'] + >>> print(Script.migrate_fact_upload.statements(variables, + ... skip_unbound=True)[-1].strip()) + commit + +''' + +from itertools import groupby +from typing import Dict, Iterable, List, Optional, Sequence, Text, Tuple, Type +from zlib import adler32 +import enum +import re +import abc + +import pkg_resources as pkg + +import sql_syntax +from sql_syntax import ( + Environment, StatementInContext, ObjectId, SQL, Name, + iter_statement) + +I2B2STAR = 'I2B2STAR' # cf. &&I2B2STAR in sql_scripts +CMS_RIF = 'CMS_RIF' + +ScriptStep = Tuple[int, Text, SQL] +Filename = str + + +class SQLMixin(enum.Enum): + @property + def sql(self) -> SQL: + from typing import cast + return cast(SQL, self.value) # hmm... + + @property + def fname(self) -> str: + return self.name + self.extension + + @abc.abstractproperty + def extension(self) -> str: + raise NotImplementedError + + @abc.abstractmethod + def parse(self, text: SQL) -> Iterable[StatementInContext]: + raise NotImplementedError + + def each_statement(self, + variables: Optional[Environment]=None, + skip_unbound: bool=False) -> Iterable[ScriptStep]: + for line, comment, statement in self.parse(self.sql): + try: + ss = sql_syntax.substitute(statement, self._all_vars(variables)) + except KeyError: + if skip_unbound: + continue + else: + raise + yield line, comment, ss + + def _all_vars(self, variables: Optional[Environment]) -> Optional[Environment]: + '''Add design_digest to variables. + ''' + if variables is None: + return None + return dict(variables, design_digest=str(self.digest())) + + def statements(self, + variables: Optional[Environment]=None, + skip_unbound: bool=False) -> Sequence[Text]: + return list(stmt for _l, _c, stmt + in self.each_statement(skip_unbound=skip_unbound, + variables=variables)) + + def created_objects(self) -> List[ObjectId]: + return [] + + def inserted_tables(self, + variables: Environment={}) -> List[Name]: + return [] + + @property + def title(self) -> Text: + line1 = self.sql.split('\n', 1)[0] + if not (line1.startswith('/** ') and ' - ' in line1): + raise ValueError('%s missing title block' % self) + return line1.split(' - ', 1)[1].strip() + + def deps(self) -> List['SQLMixin']: + return [child + for sql in self.statements() + for child in Script._get_deps(sql)] + + def dep_closure(self) -> List['SQLMixin']: + return [self] + [descendant + for sql in self.statements() + for child in Script._get_deps(sql) + for descendant in child.dep_closure()] + + def digest(self) -> int: + '''Hash the text of this script and its dependencies. + + Unlike the python hash() function, this digest is consistent across runs. + ''' + return adler32(str(self._text()).encode('utf-8')) + + def _text(self) -> List[str]: + '''Get the text of this script and its dependencies. + + >>> nodeps = Script.migrate_fact_upload + >>> nodeps._text() == [nodeps.value] + True + + >>> complex = Script.epic_flowsheets_transform + >>> complex._text() != [complex.value] + True + ''' + return sorted(set(s.sql for s in self.dep_closure())) + + @classmethod + def _get_deps(cls, sql: Text) -> List['SQLMixin']: + ''' + >>> ds = Script._get_deps( + ... "select col from t where 'dep' = 'oops.sql'") + Traceback (most recent call last): + ... + KeyError: 'oops' + + >>> Script._get_deps( + ... "select col from t where x = 'name.sql'") + [] + ''' + from typing import cast + + m = re.search(r"select \S+ from \S+ where 'dep' = '([^']+)'", sql) + if not m: + return [] + name, ext = m.group(1).rsplit('.', 1) + choices = Script if ext == 'sql' else [] + deps = [cast(SQLMixin, s) for s in choices if s.name == name] + if not deps: + raise KeyError(name) + return deps + + @classmethod + def sqlerror(cls, s: SQL) -> Optional[bool]: + if s.strip().lower() == 'whenever sqlerror exit': + return False + elif s.strip().lower() == 'whenever sqlerror continue': + return True + return None + + +class ScriptMixin(SQLMixin): + @property + def extension(self) -> str: + return '.sql' + + def parse(self, text: SQL, + block_sep: str='/') -> Iterable[StatementInContext]: + lines = [l.strip() for l in text.split('\n')] + return (sql_syntax.iter_blocks(text) if block_sep in lines + else iter_statement(text)) + + def created_objects(self) -> List[ObjectId]: + return [obj + for _l, _comment, stmt in iter_statement(self.sql) + for obj in sql_syntax.created_objects(stmt)] + + def inserted_tables(self, + variables: Optional[Environment]={}) -> List[Name]: + return [obj + for _l, _comment, stmt in iter_statement(self.sql) + for obj in sql_syntax.inserted_tables( + sql_syntax.substitute(stmt, self._all_vars(variables)))] + + +class Script(ScriptMixin, enum.Enum): + '''Script is an enum.Enum of contents. + + ISSUE: It's tempting to consider separate libraries for NAACCR, + NTDS, etc., but that doesn't integrate well with the + generic luigi.EnumParameter() in etl_tasks.SqlScriptTask. + + ''' + [ + # Keep sorted + PCORNetInit, + epic_facts_load, + epic_flowsheets_transform, + etl_tests_init, + migrate_fact_upload, + ] = [ + pkg.resource_string(__name__, + 'Oracle/' + fname).decode('utf-8') + for fname in [ + 'PCORNetInit.sql', + 'epic_facts_load.sql', + 'epic_flowsheets_transform.sql', + 'etl_tests_init.sql', + 'migrate_fact_upload.sql', + ] + ] + + def __repr__(self) -> str: + return '<%s(%s)>' % (self.__class__.__name__, self.name) + + +def _object_to_creators(libs: List[Type[SQLMixin]]) -> Dict[ObjectId, List[SQLMixin]]: + '''Find creator scripts for each object. + + "There can be only one." + >>> creators = _object_to_creators([Script]) + >>> [obj for obj, scripts in creators.items() + ... if len(scripts) > 1] + [] + ''' + fst = lambda pair: pair[0] + snd = lambda pair: pair[1] + + objs = sorted( + [(obj, s) + for lib in libs for s in lib + for obj in s.created_objects()], + key=fst) + by_obj = groupby(objs, key=fst) + return dict((obj, list(map(snd, places))) for obj, places in by_obj) + + +_redefined_objects = [ + obj for obj, scripts in _object_to_creators([Script]).items() + if len(scripts) > 1] +assert _redefined_objects == [], _redefined_objects diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3c3a387 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,49 @@ +# Usage: +# +# $ nosetests && flake8 . && mypy . + + +[nosetests] +# verbosity=3 +with-doctest=1 +where=. + +[flake8] +# E731: Assigning to lambda seems OK +# E126: Emacs python mode seems to over-indent? +ignore = E731, E126 +# with type annotations, 79 is awkward +# guide, for window sizing: +# 23456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 +max-line-length = 120 +exclude = pythonjsonlogger + + +[mypy] +# http://mypy.readthedocs.io/en/latest/config_file.html +mypy_path=stubs +warn_redundant_casts=true +warn_unused_ignores=true +strict_optional=true + +strict_boolean=false +# but all the other --strict flags (from mypy -h): +disallow_untyped_calls=true +disallow_untyped_defs=true +check_untyped_defs=true +warn_return_any=true + +[mypy-luigi.*,cx_Oracle.*,sqlalchemy.*] +disallow_untyped_defs=false + +[mypy-spreadsync.*] +ignore_errors = True + +[mypy-sqlalchemy.*] +ignore_errors = True + +[mypy-pythonjsonlogger.*] +ignore_errors = True + +#--cobertura-xml-report DIR +#--junit-xml JUNIT_XML diff --git a/sql-style.xml b/sql-style.xml new file mode 100644 index 0000000..6e1e56a --- /dev/null +++ b/sql-style.xml @@ -0,0 +1,453 @@ + + + + + 1:kumc-bmi-sql-style + + false + false + false + true + true + true + false + false + 1 + true + false + false + false + true + true + false + true + true + true + true + false + true + true + false + 2 + false + false + 3 + false + false + true + 10 + 120 + + false + kumc-bmi-sql-style + 1 + 2 + 2 + 2 + 0 + false + 60 + 1 + 0 + 0 + false + false + 0 + 3 + false + + + + 1:Old Preferences + + true + false + true + true + true + true + false + true + 0 + false + false + true + false + true + false + false + true + true + true + true + true + true + true + false + 0 + false + false + 0 + false + true + false + 999 + 999 + + false + Old Preferences + 1 + 2 + 0 + 0 + 0 + false + 80 + 1 + 0 + 0 + false + false + 0 + 0 + false + + + + 1:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 1 + 0 + 0 + false + false + 0 + 0 + false + + + + 2:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 2 + 0 + 0 + false + false + 0 + 0 + false + + + + 3:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 3 + 0 + 0 + false + false + 0 + 0 + false + + + + 4:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 4 + 0 + 0 + false + false + 0 + 0 + false + + + + 5:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 5 + 0 + 0 + false + false + 0 + 0 + false + + + + 6:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 6 + 0 + 0 + false + false + 0 + 0 + false + + + + diff --git a/sql_syntax.py b/sql_syntax.py new file mode 100644 index 0000000..e44d11c --- /dev/null +++ b/sql_syntax.py @@ -0,0 +1,238 @@ +'''sql_syntax - break SQL scripts into statements, etc. + +''' + +from datetime import datetime +from typing import Dict, Iterable, List, Optional, Text, Tuple, Union +import re + +Name = Text +SQL = Text +Environment = Dict[Name, Text] +Params = Dict[str, Union[str, int, datetime]] +Line = int +Comment = Text +StatementInContext = Tuple[Line, Comment, SQL] + + +def iter_statement(txt: SQL) -> Iterable[StatementInContext]: + r'''Iterate over SQL statements in a script. + + >>> list(iter_statement("drop table foo; create table foo")) + [(1, '', 'drop table foo'), (1, '', 'create table foo')] + + >>> list(iter_statement("-- blah blah\ndrop table foo")) + [(2, '-- blah blah\n', 'drop table foo')] + + >>> list(iter_statement("drop /* blah blah */ table foo")) + [(1, '', 'drop table foo')] + + >>> list(iter_statement("select '[^;]+' from dual")) + [(1, '', "select '[^;]+' from dual")] + + >>> list(iter_statement('select "x--y" from z;')) + [(1, '', 'select "x--y" from z')] + >>> list(iter_statement("select 'x--y' from z;")) + [(1, '', "select 'x--y' from z")] + ''' + + statement = comment = '' + line = 1 + sline = None # type: Optional[int] + + def save(txt: SQL) -> Tuple[SQL, Optional[int]]: + return (statement + txt, sline or (line if txt else None)) + + while 1: + m = SQL_SEPARATORS.search(txt) + if not m: + statement, sline = save(txt) + break + + pfx, match, txt = (txt[:m.start()], + txt[m.start():m.end()], + txt[m.end():]) + if pfx: + statement, sline = save(pfx) + + if m.group('sep'): + if sline: + yield sline, comment, statement + statement = comment = '' + sline = None + elif [n for n in ('lit', 'hint', 'sym') + if m.group(n)]: + statement, sline = save(match) + elif (m.group('space') and statement): + statement, sline = save(match) + elif ((m.group('comment') and not statement) or + (m.group('space') and comment)): + comment += match + + line += (pfx + match).count("\n") + + if sline and (comment or statement): + yield sline, comment, statement + + +# Check for hint before comment since a hint looks like a comment +SQL_SEPARATORS = re.compile( + r'(?P^\s+)' + r'|(?P/\*\+.*?\*/)' + r'|(?P(--[^\n]*(?:\n|$))|(?:/\*([^\*]|(\*(?!/)))*\*/))' + r'|(?P"[^\"]*")' + r"|(?P'[^\']*')" + r'|(?P;)') + + +def _test_iter_statement() -> None: + r''' + >>> list(iter_statement("/* blah blah */ drop table foo")) + [(1, '/* blah blah */ ', 'drop table foo')] + + >>> [l for (l, c, s) in iter_statement('s1;\n/**********************\n' + ... '* Medication concepts \n' + ... '**********************/\ns2')] + [1, 5] + + >>> list(iter_statement("")) + [] + + >>> list(iter_statement("/*...*/ ")) + [] + + >>> list(iter_statement("drop table foo; ")) + [(1, '', 'drop table foo')] + + >>> list(iter_statement("drop table foo")) + [(1, '', 'drop table foo')] + + >>> list(iter_statement("/* *** -- *** */ drop table foo")) + [(1, '/* *** -- *** */ ', 'drop table foo')] + + >>> list(iter_statement("/* *** -- *** */ drop table x /* ... */")) + [(1, '/* *** -- *** */ ', 'drop table x ')] + + >>> list(iter_statement('select /*+ index(ix1 ix2) */ * from some_table')) + [(1, '', 'select /*+ index(ix1 ix2) */ * from some_table')] + + >>> len(list(iter_statement(""" /* + no space before + in hints*/ + ... /*+ index(observation_fact fact_cnpt_pat_enct_idx) */ + ... select * from some_table; """))[0][2]) + 79 + + >>> list(iter_statement('select 1+1; /* nothing else */; ')) + [(1, '', 'select 1+1')] + ''' + pass # pragma: nocover + + +def substitute(sql: SQL, variables: Optional[Environment]) -> SQL: + '''Evaluate substitution variables in the style of Oracle sqlplus. + + >>> substitute('select &¬_bound from dual', {}) + Traceback (most recent call last): + KeyError: 'not_bound' + ''' + if variables is None: + return sql + sql_esc = sql.replace('%', '%%') # escape %, which we use specially + return re.sub('&&(\w+)', r'%(\1)s', sql_esc) % variables + + +def params_used(params: Params, statement: SQL) -> Params: + return dict((k, v) for (k, v) in params.items() + if k in param_names(statement)) + + +def param_names(s: SQL) -> List[Name]: + ''' + >>> param_names('select 1+1 from dual') + [] + >>> param_names('select 1+:y from dual') + ['y'] + ''' + return [expr[1:] + for expr in re.findall(r':\w+', s)] + + +def first_cursor(statement: SQL) -> SQL: + '''Find argument of first obvious call to `cursor()`. + + >>> first_cursor('select * from table(f(cursor(select * from there)))') + 'select * from there' + ''' + return statement.split('cursor(')[1].split(')')[0] + + +class ObjectId(object): + kind = '' + + def __init__(self, name: str) -> None: + self.name = name + + def __repr__(self) -> str: + return '{} {}'.format(self.kind, self.name) + + def __hash__(self) -> int: + return hash((self.kind, self.name)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ObjectId): + return False + return (self.kind, self.name) == ((other.kind, other.name)) + + def __lt__(self, other: 'ObjectId') -> bool: + return (self.kind, self.name) < ((other.kind, other.name)) + + +class TableId(ObjectId): + kind = 'table' + + +class ViewId(ObjectId): + kind = 'view' + + +def created_objects(statement: SQL) -> List[ObjectId]: + r''' + >>> created_objects('create table t as ...') + [table t] + + >>> created_objects('create or replace view x\nas ...') + [view x] + ''' + m = re.search('^create or replace view (\S+)', statement.strip()) + views = [ViewId(m.group(1))] if m else [] # type: List[ObjectId] + m = re.search('^create table (\S+)', statement.strip()) + tables = [TableId(m.group(1))] if m else [] # type: List[ObjectId] + return tables + views + + +def inserted_tables(statement: SQL) -> List[Name]: + r''' + >>> inserted_tables('create table t as ...') + [] + >>> inserted_tables('insert into t (...) ...') + ['t'] + ''' + if not statement.startswith('insert'): + return [] + m = re.search('into\s+(\S+)', statement.strip()) + return [m.group(1)] if m else [] + + +def insert_append_table(statement: SQL) -> Optional[Name]: + if '/*+ append' in statement: + [t] = inserted_tables(statement) + return t + return None + + +def iter_blocks(module: SQL, + separator: str =r'\r?\n/\r?\n') -> Iterable[StatementInContext]: + line = 1 + for block in re.split(separator, module): + if block.strip(): + yield (line, '...no comment handling...', block) + line += len((block + separator).split('\n')[:-1]) diff --git a/stubs/cx_Oracle.pyi b/stubs/cx_Oracle.pyi new file mode 100644 index 0000000..10a8948 --- /dev/null +++ b/stubs/cx_Oracle.pyi @@ -0,0 +1,429 @@ +# Stubs for cx_Oracle (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from datetime import date +from typing import Any + +ATTR_PURITY_DEFAULT = ... # type: int +ATTR_PURITY_NEW = ... # type: int +ATTR_PURITY_SELF = ... # type: int +DBSHUTDOWN_ABORT = ... # type: int +DBSHUTDOWN_FINAL = ... # type: int +DBSHUTDOWN_IMMEDIATE = ... # type: int +DBSHUTDOWN_TRANSACTIONAL = ... # type: int +DBSHUTDOWN_TRANSACTIONAL_LOCAL = ... # type: int +EVENT_DEREG = ... # type: int +EVENT_NONE = ... # type: int +EVENT_OBJCHANGE = ... # type: int +EVENT_QUERYCHANGE = ... # type: int +EVENT_SHUTDOWN = ... # type: int +EVENT_SHUTDOWN_ANY = ... # type: int +EVENT_STARTUP = ... # type: int +FNCODE_BINDBYNAME = ... # type: int +FNCODE_BINDBYPOS = ... # type: int +FNCODE_DEFINEBYPOS = ... # type: int +FNCODE_STMTEXECUTE = ... # type: int +FNCODE_STMTFETCH = ... # type: int +FNCODE_STMTPREPARE = ... # type: int +OPCODE_ALLOPS = ... # type: int +OPCODE_ALLROWS = ... # type: int +OPCODE_ALTER = ... # type: int +OPCODE_DELETE = ... # type: int +OPCODE_DROP = ... # type: int +OPCODE_INSERT = ... # type: int +OPCODE_UPDATE = ... # type: int +PRELIM_AUTH = ... # type: int +SPOOL_ATTRVAL_FORCEGET = ... # type: int +SPOOL_ATTRVAL_NOWAIT = ... # type: int +SPOOL_ATTRVAL_WAIT = ... # type: int +SUBSCR_CQ_QOS_BEST_EFFORT = ... # type: int +SUBSCR_CQ_QOS_CLQRYCACHE = ... # type: int +SUBSCR_CQ_QOS_QUERY = ... # type: int +SUBSCR_NAMESPACE_DBCHANGE = ... # type: int +SUBSCR_PROTO_HTTP = ... # type: int +SUBSCR_PROTO_MAIL = ... # type: int +SUBSCR_PROTO_OCI = ... # type: int +SUBSCR_PROTO_SERVER = ... # type: int +SUBSCR_QOS_HAREG = ... # type: int +SUBSCR_QOS_MULTICBK = ... # type: int +SUBSCR_QOS_PAYLOAD = ... # type: int +SUBSCR_QOS_PURGE_ON_NTFN = ... # type: int +SUBSCR_QOS_RELIABLE = ... # type: int +SUBSCR_QOS_REPLICATE = ... # type: int +SUBSCR_QOS_SECURE = ... # type: int +SYSASM = ... # type: int +SYSDBA = ... # type: int +SYSOPER = ... # type: int +UCBTYPE_ENTRY = ... # type: int +UCBTYPE_EXIT = ... # type: int +UCBTYPE_REPLACE = ... # type: int +apilevel = ... # type: str +buildtime = ... # type: str +paramstyle = ... # type: str +threadsafety = ... # type: int +version = ... # type: str + +def DateFromTicks(*args, **kwargs): ... +def Time(*args, **kwargs): ... +def TimeFromTicks(*args, **kwargs): ... +def TimestampFromTicks(*args, **kwargs): ... +def clientversion(*args, **kwargs): ... +def makedsn(*args, **kwargs): ... + +class _BASEVARTYPE(object): ... + +class BFILE(_BASEVARTYPE): ... + +class BINARY(_BASEVARTYPE): ... + +class BLOB(_BASEVARTYPE): ... + +class Binary: + maketrans = ... # type: Any + def __init__(self, *args, **kwargs): ... + def capitalize(self, *args, **kwargs): ... + def center(self, *args, **kwargs): ... + def count(self, *args, **kwargs): ... + def decode(self, *args, **kwargs): ... + def endswith(self, *args, **kwargs): ... + def expandtabs(self, *args, **kwargs): ... + def find(self, *args, **kwargs): ... + @classmethod + def fromhex(cls, *args, **kwargs): ... + def hex(self, *args, **kwargs): ... + def index(self, *args, **kwargs): ... + def isalnum(self, *args, **kwargs): ... + def isalpha(self, *args, **kwargs): ... + def isdigit(self, *args, **kwargs): ... + def islower(self, *args, **kwargs): ... + def isspace(self, *args, **kwargs): ... + def istitle(self, *args, **kwargs): ... + def isupper(self, *args, **kwargs): ... + def join(self, *args, **kwargs): ... + def ljust(self, *args, **kwargs): ... + def lower(self, *args, **kwargs): ... + def lstrip(self, *args, **kwargs): ... + def partition(self, *args, **kwargs): ... + def replace(self, *args, **kwargs): ... + def rfind(self, *args, **kwargs): ... + def rindex(self, *args, **kwargs): ... + def rjust(self, *args, **kwargs): ... + def rpartition(self, *args, **kwargs): ... + def rsplit(self, *args, **kwargs): ... + def rstrip(self, *args, **kwargs): ... + def split(self, *args, **kwargs): ... + def splitlines(self, *args, **kwargs): ... + def startswith(self, *args, **kwargs): ... + def strip(self, *args, **kwargs): ... + def swapcase(self, *args, **kwargs): ... + def title(self, *args, **kwargs): ... + def translate(self, *args, **kwargs): ... + def upper(self, *args, **kwargs): ... + def zfill(self, *args, **kwargs): ... + def __add__(self, other): ... + def __contains__(self, *args, **kwargs): ... + def __eq__(self, other): ... + def __ge__(self, other): ... + def __getitem__(self, index): ... + def __getnewargs__(self, *args, **kwargs): ... + def __gt__(self, other): ... + def __hash__(self): ... + def __iter__(self): ... + def __le__(self, other): ... + def __len__(self, *args, **kwargs): ... + def __lt__(self, other): ... + def __mod__(self, other): ... + def __mul__(self, other): ... + def __ne__(self, other): ... + def __rmod__(self, other): ... + def __rmul__(self, other): ... + +class CLOB(_BASEVARTYPE): ... + +class CURSOR(_BASEVARTYPE): ... + +class Connection: + action = ... # type: Any + autocommit = ... # type: Any + client_identifier = ... # type: Any + clientinfo = ... # type: Any + current_schema = ... # type: Any + dsn = ... # type: Any + encoding = ... # type: Any + inputtypehandler = ... # type: Any + maxBytesPerCharacter = ... # type: Any + module = ... # type: Any + nencoding = ... # type: Any + outputtypehandler = ... # type: Any + stmtcachesize = ... # type: Any + tnsentry = ... # type: Any + username = ... # type: Any + version = ... # type: Any + def __init__(self, *args, **kwargs): ... + def begin(self, *args, **kwargs): ... + def cancel(self, *args, **kwargs): ... + def changepassword(self, *args, **kwargs): ... + def close(self, *args, **kwargs): ... + def commit(self, *args, **kwargs): ... + def cursor(self, *args, **kwargs): ... + def ping(self, *args, **kwargs): ... + def prepare(self, *args, **kwargs): ... + def register(self, *args, **kwargs): ... + def rollback(self, *args, **kwargs): ... + def shutdown(self, *args, **kwargs): ... + def startup(self, *args, **kwargs): ... + def subscribe(self, *args, **kwargs): ... + def unregister(self, *args, **kwargs): ... + def __enter__(self, *args, **kwargs): ... + def __exit__(self, *args, **kwargs): ... + +class Cursor: + arraysize = ... # type: Any + bindarraysize = ... # type: Any + bindvars = ... # type: Any + connection = ... # type: Any + description = ... # type: Any + fetchvars = ... # type: Any + inputtypehandler = ... # type: Any + numbersAsStrings = ... # type: Any + outputtypehandler = ... # type: Any + rowcount = ... # type: Any + rowfactory = ... # type: Any + statement = ... # type: Any + def __init__(self, *args, **kwargs): ... + def arrayvar(self, *args, **kwargs): ... + def bindnames(self, *args, **kwargs): ... + def callfunc(self, *args, **kwargs): ... + def callproc(self, *args, **kwargs): ... + def close(self, *args, **kwargs): ... + def execute(self, *args, **kwargs): ... + def executemany(self, *args, **kwargs): ... + def executemanyprepared(self, *args, **kwargs): ... + def fetchall(self, *args, **kwargs): ... + def fetchmany(self, *args, **kwargs): ... + def fetchone(self, *args, **kwargs): ... + def fetchraw(self, *args, **kwargs): ... + def getbatcherrors(self, *args, **kwargs): ... + def parse(self, *args, **kwargs): ... + def prepare(self, *args, **kwargs): ... + def setinputsizes(self, *args, **kwargs): ... + def setoutputsize(self, *args, **kwargs): ... + def var(self, *args, **kwargs): ... + def __iter__(self): ... + def __next__(self): ... + +class DATETIME(_BASEVARTYPE): ... + +class DataError(DatabaseError): ... + +class DatabaseError(Error): ... + +class Date: + day = ... # type: Any + max = ... # type: Any + min = ... # type: Any + month = ... # type: Any + resolution = ... # type: Any + year = ... # type: Any + def __init__(self, *args, **kwargs): ... + def ctime(self, *args, **kwargs): ... + @classmethod + def fromordinal(cls, *args, **kwargs): ... + @classmethod + def fromtimestamp(cls, *args, **kwargs): ... + def isocalendar(self, *args, **kwargs): ... + def isoformat(self, *args, **kwargs): ... + def isoweekday(self, *args, **kwargs): ... + def replace(self, *args, **kwargs): ... + def strftime(self, *args, **kwargs): ... + def timetuple(self, *args, **kwargs): ... + @classmethod + def today(cls, *args, **kwargs): ... + def toordinal(self, *args, **kwargs): ... + def weekday(self, *args, **kwargs): ... + def __add__(self, other): ... + def __eq__(self, other): ... + def __format__(self, *args, **kwargs): ... + def __ge__(self, other): ... + def __gt__(self, other): ... + def __hash__(self): ... + def __le__(self, other): ... + def __lt__(self, other): ... + def __ne__(self, other): ... + def __radd__(self, other): ... + def __reduce__(self): ... + def __rsub__(self, other): ... + def __sub__(self, other): ... + +class Error(Exception): ... + +class FIXED_CHAR(_BASEVARTYPE): ... + +class FIXED_NCHAR(_BASEVARTYPE): ... + +class FIXED_UNICODE(_BASEVARTYPE): ... + +class INTERVAL(_BASEVARTYPE): ... + +class IntegrityError(DatabaseError): ... + +class InterfaceError(Error): ... + +class InternalError(DatabaseError): ... + +class LOB: + def close(self, *args, **kwargs): ... + def fileexists(self, *args, **kwargs): ... + def getchunksize(self, *args, **kwargs): ... + def getfilename(self, *args, **kwargs): ... + def isopen(self, *args, **kwargs): ... + def open(self, *args, **kwargs): ... + def read(self, *args, **kwargs): ... + def setfilename(self, *args, **kwargs): ... + def size(self, *args, **kwargs): ... + def trim(self, *args, **kwargs): ... + def write(self, *args, **kwargs): ... + def __reduce__(self): ... + +class LONG_BINARY(_BASEVARTYPE): ... + +class LONG_NCHAR(_BASEVARTYPE): ... + +class LONG_STRING(_BASEVARTYPE): ... + +class LONG_UNICODE(_BASEVARTYPE): ... + +class NATIVE_FLOAT(_BASEVARTYPE): ... + +class NCHAR(_BASEVARTYPE): ... + +class NCLOB(_BASEVARTYPE): ... + +class NUMBER(_BASEVARTYPE): ... + +class NotSupportedError(DatabaseError): ... + +class OBJECT(_BASEVARTYPE): + type = ... # type: Any + +class OperationalError(DatabaseError): ... + +class ProgrammingError(DatabaseError): ... + +class ROWID(_BASEVARTYPE): ... + +class STRING(_BASEVARTYPE): ... + +class SessionPool: + busy = ... # type: Any + dsn = ... # type: Any + getmode = ... # type: Any + homogeneous = ... # type: Any + increment = ... # type: Any + max = ... # type: Any + min = ... # type: Any + name = ... # type: Any + opened = ... # type: Any + timeout = ... # type: Any + tnsentry = ... # type: Any + username = ... # type: Any + def __init__(self, *args, **kwargs): ... + def acquire(self, *args, **kwargs): ... + def drop(self, *args, **kwargs): ... + def release(self, *args, **kwargs): ... + +class TIMESTAMP(_BASEVARTYPE): ... + +class Timestamp(date): + hour = ... # type: Any + max = ... # type: Any + microsecond = ... # type: Any + min = ... # type: Any + minute = ... # type: Any + resolution = ... # type: Any + second = ... # type: Any + tzinfo = ... # type: Any + def __init__(self, *args, **kwargs): ... + def astimezone(self, *args, **kwargs): ... + @classmethod + def combine(cls, *args, **kwargs): ... + def ctime(self, *args, **kwargs): ... + def date(self, *args, **kwargs): ... + def dst(self, *args, **kwargs): ... + @classmethod + def fromtimestamp(cls, *args, **kwargs): ... + def isoformat(self, *args, **kwargs): ... + @classmethod + def now(cls, *args, **kwargs): ... + def replace(self, *args, **kwargs): ... + @classmethod + def strptime(cls, *args, **kwargs): ... + def time(self, *args, **kwargs): ... + def timestamp(self, *args, **kwargs): ... + def timetuple(self, *args, **kwargs): ... + def timetz(self, *args, **kwargs): ... + def tzname(self, *args, **kwargs): ... + @classmethod + def utcfromtimestamp(cls, *args, **kwargs): ... + @classmethod + def utcnow(cls, *args, **kwargs): ... + def utcoffset(self, *args, **kwargs): ... + def utctimetuple(self, *args, **kwargs): ... + def __add__(self, other): ... + def __eq__(self, other): ... + def __ge__(self, other): ... + def __gt__(self, other): ... + def __hash__(self): ... + def __le__(self, other): ... + def __lt__(self, other): ... + def __ne__(self, other): ... + def __radd__(self, other): ... + def __reduce__(self): ... + def __rsub__(self, other): ... + def __sub__(self, other): ... + +class UNICODE(_BASEVARTYPE): ... + +class Warning(Exception): ... + +class _Error: + code = ... # type: Any + context = ... # type: Any + message = ... # type: Any + offset = ... # type: Any + +class connect: + action = ... # type: Any + autocommit = ... # type: Any + client_identifier = ... # type: Any + clientinfo = ... # type: Any + current_schema = ... # type: Any + dsn = ... # type: Any + encoding = ... # type: Any + inputtypehandler = ... # type: Any + maxBytesPerCharacter = ... # type: Any + module = ... # type: Any + nencoding = ... # type: Any + outputtypehandler = ... # type: Any + stmtcachesize = ... # type: Any + tnsentry = ... # type: Any + username = ... # type: Any + version = ... # type: Any + def __init__(self, *args, **kwargs): ... + def begin(self, *args, **kwargs): ... + def cancel(self, *args, **kwargs): ... + def changepassword(self, *args, **kwargs): ... + def close(self, *args, **kwargs): ... + def commit(self, *args, **kwargs): ... + def cursor(self, *args, **kwargs): ... + def ping(self, *args, **kwargs): ... + def prepare(self, *args, **kwargs): ... + def register(self, *args, **kwargs): ... + def rollback(self, *args, **kwargs): ... + def shutdown(self, *args, **kwargs): ... + def startup(self, *args, **kwargs): ... + def subscribe(self, *args, **kwargs): ... + def unregister(self, *args, **kwargs): ... + def __enter__(self, *args, **kwargs): ... + def __exit__(self, *args, **kwargs): ... diff --git a/stubs/luigi/__init__.pyi b/stubs/luigi/__init__.pyi new file mode 100644 index 0000000..3a21420 --- /dev/null +++ b/stubs/luigi/__init__.pyi @@ -0,0 +1,20 @@ +# Stubs for luigi (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from luigi import task as task +from luigi.task import Task as Task, Config as Config, ExternalTask as ExternalTask, WrapperTask as WrapperTask, namespace as namespace, auto_namespace as auto_namespace +from luigi import target as target +from luigi.target import Target as Target +from luigi import local_target as local_target +from luigi.local_target import LocalTarget as LocalTarget +from luigi import rpc as rpc +from luigi.rpc import RemoteScheduler as RemoteScheduler, RPCError as RPCError +from luigi import parameter as parameter +from luigi.parameter import Parameter as Parameter, DateParameter as DateParameter, MonthParameter as MonthParameter, YearParameter as YearParameter, DateHourParameter as DateHourParameter, DateMinuteParameter as DateMinuteParameter, DateSecondParameter as DateSecondParameter, DateIntervalParameter as DateIntervalParameter, TimeDeltaParameter as TimeDeltaParameter, IntParameter as IntParameter, FloatParameter as FloatParameter, BoolParameter as BoolParameter, TaskParameter as TaskParameter, EnumParameter as EnumParameter, DictParameter as DictParameter, ListParameter as ListParameter, TupleParameter as TupleParameter, NumericalParameter as NumericalParameter, ChoiceParameter as ChoiceParameter +from luigi import configuration as configuration +from luigi import interface as interface +from luigi.interface import run as run, build as build +from luigi import event as event +from luigi.event import Event as Event +from .tools import range as range diff --git a/stubs/luigi/configuration.pyi b/stubs/luigi/configuration.pyi new file mode 100644 index 0000000..06a65a2 --- /dev/null +++ b/stubs/luigi/configuration.pyi @@ -0,0 +1,26 @@ +# Stubs for luigi.configuration (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional +# py2: from ConfigParser import ConfigParser +from configparser import ConfigParser + +class LuigiConfigParser(ConfigParser): + NO_DEFAULT = ... # type: Any + config_file = ... # type: Any + @classmethod + def add_config_path(cls, path): ... + @classmethod + def instance(cls, *args, **kwargs): ... + @classmethod + def reload(cls): ... + # avoid: Signature of "get" incompatible with supertype "Mapping" + def get(self, section, option, default: Any = ..., **kwargs): ... # type:ignore + def getboolean(self, section, option, default: Any = ...): ... # type:ignore + def getint(self, section, option, default: Any = ...): ... # type:ignore + def getfloat(self, section, option, default: Any = ...): ... # type:ignore + def getintdict(self, section): ... + def set(self, section, option, value: Optional[Any] = ...): ... + +def get_config(): ... diff --git a/stubs/luigi/contrib/__init__.pyi b/stubs/luigi/contrib/__init__.pyi new file mode 100644 index 0000000..1a7b73e --- /dev/null +++ b/stubs/luigi/contrib/__init__.pyi @@ -0,0 +1,4 @@ +# Stubs for luigi.contrib (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + diff --git a/stubs/luigi/contrib/sqla.pyi b/stubs/luigi/contrib/sqla.pyi new file mode 100644 index 0000000..eb90250 --- /dev/null +++ b/stubs/luigi/contrib/sqla.pyi @@ -0,0 +1,43 @@ +# Stubs for luigi.contrib.sqla (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Dict, Any +import luigi +from collections import namedtuple +from sqlalchemy.engine import Engine + +class SQLAlchemyTarget(luigi.Target): + marker_table = ... # type: Any + + Connection = namedtuple('Connection', 'engine pid') + target_table = ... # type: str + update_id = ... # type: str + connection_string = ... # type: str + echo = ... # type: bool + connect_args = ... # type: Dict[str, Any] + marker_table_bound = ... # type: Any + def __init__(self, connection_string, target_table, update_id, echo: bool = ..., connect_args: Any = ...) -> None: ... + @property + def engine(self) -> Engine: ... + def touch(self) -> None: ... + def exists(self) -> bool: ... + def create_marker_table(self) -> None: ... + def open(self, mode): ... + +class CopyToTable(luigi.Task): + echo = ... # type: bool + connect_args = ... # type: Any + def connection_string(self): ... + def table(self): ... + columns = ... # type: Any + column_separator = ... # type: str + chunk_size = ... # type: int + reflect = ... # type: bool + table_bound = ... # type: Any + def create_table(self, engine): ... + def update_id(self): ... + def output(self): ... + def rows(self): ... + def run(self): ... + def copy(self, conn, ins_rows, table_bound): ... diff --git a/stubs/luigi/event.pyi b/stubs/luigi/event.pyi new file mode 100644 index 0000000..1cfa689 --- /dev/null +++ b/stubs/luigi/event.pyi @@ -0,0 +1,15 @@ +# Stubs for luigi.event (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +class Event: + DEPENDENCY_DISCOVERED = ... # type: str + DEPENDENCY_MISSING = ... # type: str + DEPENDENCY_PRESENT = ... # type: str + BROKEN_TASK = ... # type: str + START = ... # type: str + FAILURE = ... # type: str + SUCCESS = ... # type: str + PROCESSING_TIME = ... # type: str + TIMEOUT = ... # type: str + PROCESS_FAILURE = ... # type: str diff --git a/stubs/luigi/interface.pyi b/stubs/luigi/interface.pyi new file mode 100644 index 0000000..950c658 --- /dev/null +++ b/stubs/luigi/interface.pyi @@ -0,0 +1,37 @@ +# Stubs for luigi.interface (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional +from . import task + +def setup_interface_logging(conf_file: str = ..., level_name: str = ...): ... + +class core(task.Config): + use_cmdline_section = ... # type: bool + local_scheduler = ... # type: Any + scheduler_host = ... # type: Any + scheduler_port = ... # type: Any + scheduler_url = ... # type: Any + lock_size = ... # type: Any + no_lock = ... # type: Any + lock_pid_dir = ... # type: Any + take_lock = ... # type: Any + workers = ... # type: Any + logging_conf_file = ... # type: Any + log_level = ... # type: Any + module = ... # type: Any + parallel_scheduling = ... # type: Any + assistant = ... # type: Any + help = ... # type: Any + help_all = ... # type: Any + +class _WorkerSchedulerFactory: + def create_local_scheduler(self): ... + def create_remote_scheduler(self, url): ... + def create_worker(self, scheduler, worker_processes, assistant: bool = ...): ... + +class PidLockAlreadyTakenExit(SystemExit): ... + +def run(*args, **kwargs): ... +def build(tasks, worker_scheduler_factory: Optional[Any] = ..., **env_params): ... diff --git a/stubs/luigi/local_target.pyi b/stubs/luigi/local_target.pyi new file mode 100644 index 0000000..d3883e4 --- /dev/null +++ b/stubs/luigi/local_target.pyi @@ -0,0 +1,35 @@ +# Stubs for luigi.local_target (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional +from luigi.target import FileSystem, FileSystemTarget, AtomicLocalFile + +class atomic_file(AtomicLocalFile): + def move_to_final_destination(self): ... + def generate_tmp_path(self, path): ... + +class LocalFileSystem(FileSystem): + def copy(self, old_path, new_path, raise_if_exists: bool = ...): ... + def exists(self, path): ... + def mkdir(self, path, parents: bool = ..., raise_if_exists: bool = ...): ... + def isdir(self, path): ... + def listdir(self, path): ... + # Signature of "remove" incompatible with supertype "FileSystem" + def remove(self, path, recursive: bool = ...): ... # type: ignore + def move(self, old_path, new_path, raise_if_exists: bool = ...): ... + +class LocalTarget(FileSystemTarget): + fs = ... # type: Any + format = ... # type: Any + is_tmp = ... # type: Any + def __init__(self, path: Optional[Any] = ..., format: Optional[Any] = ..., is_tmp: bool = ...) -> None: ... + def makedirs(self): ... + def open(self, mode: str = ...): ... + def move(self, new_path, raise_if_exists: bool = ...): ... + def move_dir(self, new_path): ... + def remove(self): ... + def copy(self, new_path, raise_if_exists: bool = ...): ... + @property + def fn(self): ... + def __del__(self): ... diff --git a/stubs/luigi/parameter.pyi b/stubs/luigi/parameter.pyi new file mode 100644 index 0000000..a550034 --- /dev/null +++ b/stubs/luigi/parameter.pyi @@ -0,0 +1,131 @@ +# Stubs for luigi.parameter (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from json.encoder import JSONEncoder +from typing import Any, Optional +from collections import Mapping + +class ParameterException(Exception): ... +class MissingParameterException(ParameterException): ... +class UnknownParameterException(ParameterException): ... +class DuplicateParameterException(ParameterException): ... + +class Parameter: + significant = ... # type: Any + positional = ... # type: Any + description = ... # type: Any + always_in_help = ... # type: Any + def __init__(self, default: Any = ..., is_global: bool = ..., significant: bool = ..., description: Optional[Any] = ..., config_path: Optional[Any] = ..., positional: bool = ..., always_in_help: bool = ..., batch_method: Optional[Any] = ...) -> None: ... + def has_task_value(self, task_name, param_name): ... + def task_value(self, task_name, param_name): ... + def parse(self, x): ... + def serialize(self, x): ... + def normalize(self, x): ... + def next_in_enumeration(self, _value): ... + +class _DateParameterBase(Parameter): + interval = ... # type: Any + start = ... # type: Any + def __init__(self, interval: int = ..., start: Optional[Any] = ..., **kwargs) -> None: ... + def date_format(self): ... + def parse(self, s): ... + def serialize(self, dt): ... + +class DateParameter(_DateParameterBase): + date_format = ... # type: str + def next_in_enumeration(self, value): ... + def normalize(self, value): ... + +class MonthParameter(DateParameter): + date_format = ... # type: str + def next_in_enumeration(self, value): ... + def normalize(self, value): ... + +class YearParameter(DateParameter): + date_format = ... # type: str + def next_in_enumeration(self, value): ... + def normalize(self, value): ... + +class _DatetimeParameterBase(Parameter): + interval = ... # type: Any + start = ... # type: Any + def __init__(self, interval: int = ..., start: Optional[Any] = ..., **kwargs) -> None: ... + def date_format(self): ... + def parse(self, s): ... + def serialize(self, dt): ... + def normalize(self, dt): ... + def next_in_enumeration(self, value): ... + +class DateHourParameter(_DatetimeParameterBase): + date_format = ... # type: str + +class DateMinuteParameter(_DatetimeParameterBase): + date_format = ... # type: str + deprecated_date_format = ... # type: str + def parse(self, s): ... + +class DateSecondParameter(_DatetimeParameterBase): + date_format = ... # type: str + +class IntParameter(Parameter): + def parse(self, s): ... + def next_in_enumeration(self, value): ... + +class FloatParameter(Parameter): + def parse(self, s): ... + +class BoolParameter(Parameter): + def __init__(self, *args, **kwargs) -> None: ... + def parse(self, s): ... + def normalize(self, value): ... + +class DateIntervalParameter(Parameter): + def parse(self, s): ... + +class TimeDeltaParameter(Parameter): + def parse(self, input): ... + def serialize(self, x): ... + +class TaskParameter(Parameter): + def parse(self, input): ... + def serialize(self, cls): ... + +class EnumParameter(Parameter): + def __init__(self, *args, **kwargs) -> None: ... + def parse(self, s): ... + def serialize(self, e): ... + +class FrozenOrderedDict(Mapping): + def __init__(self, *args, **kwargs) -> None: ... + def __getitem__(self, key): ... + def __iter__(self): ... + def __len__(self): ... + def __hash__(self): ... + def get_wrapped(self): ... + +class DictParameter(Parameter): + class DictParamEncoder(JSONEncoder): + def default(self, obj): ... + def normalize(self, value): ... + def parse(self, s): ... + def serialize(self, x): ... + +class ListParameter(Parameter): + def normalize(self, x): ... + def parse(self, x): ... + def serialize(self, x): ... + +class TupleParameter(Parameter): + def parse(self, x): ... + def serialize(self, x): ... + +class NumericalParameter(Parameter): + description = ... # type: str + def __init__(self, left_op: Any = ..., right_op: Any = ..., *args, **kwargs) -> None: ... + def parse(self, s): ... + +class ChoiceParameter(Parameter): + description = ... # type: str + def __init__(self, var_type: Any = ..., *args, **kwargs) -> None: ... + def parse(self, s): ... diff --git a/stubs/luigi/rpc.pyi b/stubs/luigi/rpc.pyi new file mode 100644 index 0000000..b73c49f --- /dev/null +++ b/stubs/luigi/rpc.pyi @@ -0,0 +1,26 @@ +# Stubs for luigi.rpc (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional + +HAS_UNIX_SOCKET = ... # type: bool +HAS_REQUESTS = ... # type: bool +logger = ... # type: Any + +class RPCError(Exception): + sub_exception = ... # type: Any + def __init__(self, message, sub_exception: Optional[Any] = ...) -> None: ... + +class URLLibFetcher: + raises = ... # type: Any + def fetch(self, full_url, body, timeout): ... + +class RequestsFetcher: + raises = ... # type: Any + session = ... # type: Any + def __init__(self, session) -> None: ... + def fetch(self, full_url, body, timeout): ... + +class RemoteScheduler: + def __init__(self, url: str = ..., connect_timeout: Optional[Any] = ...) -> None: ... diff --git a/stubs/luigi/target.pyi b/stubs/luigi/target.pyi new file mode 100644 index 0000000..3903aa4 --- /dev/null +++ b/stubs/luigi/target.pyi @@ -0,0 +1,47 @@ +# Stubs for luigi.target (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any +import io + +logger = ... # type: Any + +class Target: + def exists(self) -> bool: ... + +class FileSystemException(Exception): ... +class FileAlreadyExists(FileSystemException): ... +class MissingParentDirectory(FileSystemException): ... +class NotADirectory(FileSystemException): ... + +class FileSystem: + def exists(self, path): ... + def remove(self, path, recursive: bool = ..., skip_trash: bool = ...): ... + def mkdir(self, path, parents: bool = ..., raise_if_exists: bool = ...): ... + def isdir(self, path): ... + def listdir(self, path): ... + def move(self, path, dest): ... + def rename_dont_move(self, path, dest): ... + def rename(self, *args, **kwargs): ... + def copy(self, path, dest): ... + +class FileSystemTarget(Target): + path = ... # type: Any + def __init__(self, path) -> None: ... + def fs(self): ... + def open(self, mode): ... + def exists(self): ... + def remove(self): ... + def temporary_path(self): ... + +class AtomicLocalFile(io.BufferedWriter): + path = ... # type: Any + def __init__(self, path) -> None: ... + def close(self): ... + def generate_tmp_path(self, path): ... + def move_to_final_destination(self): ... + def __del__(self): ... + @property + def tmp_path(self): ... + def __exit__(self, exc_type, exc, traceback): ... diff --git a/stubs/luigi/task.pyi b/stubs/luigi/task.pyi new file mode 100644 index 0000000..66e892c --- /dev/null +++ b/stubs/luigi/task.pyi @@ -0,0 +1,107 @@ +# Stubs for luigi.task (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Dict, Generic, Iterable, List, Optional, TypeVar, Type, Union + +from .target import Target + +Parameter = ... # type: Any +logger = ... # type: Any +TASK_ID_INCLUDE_PARAMS = ... # type: int +TASK_ID_TRUNCATE_PARAMS = ... # type: int +TASK_ID_TRUNCATE_HASH = ... # type: int +TASK_ID_INVALID_CHAR_REGEX = ... # type: Any + +def namespace(namespace: Optional[Any] = ..., scope: str = ...): ... +def auto_namespace(scope: str = ...): ... +def task_id_str(task_family, params): ... + +class BulkCompleteNotImplementedError(NotImplementedError): ... + +_R = TypeVar('_R') +_I = TypeVar('_I') +_O = TypeVar('_O') +_T = TypeVar('_T') + +class Task(Generic[_R, _I, _O]): + priority = ... # type: int + disabled = ... # type: bool + resources = ... # type: Any + worker_timeout = ... # type: Any + max_batch_size = ... # type: Any + @property + def batchable(self): ... + @property + def retry_count(self): ... + @property + def disable_hard_timeout(self): ... + @property + def disable_window_seconds(self): ... + @property + def owner_email(self): ... + @property + def use_cmdline_section(self): ... + @classmethod + def event_handler(cls, event): ... + def trigger_event(self, event, *args, **kwargs): ... + @property + def task_module(self): ... + task_namespace = ... # type: Any + @classmethod + def get_task_namespace(cls): ... + @property + def task_family(self): ... + @classmethod + def get_task_family(cls): ... + @classmethod + def get_params(cls): ... + @classmethod + def batch_param_names(cls): ... + @classmethod + def get_param_names(cls, include_significant: bool = ...) -> List[str]: ... + @classmethod + def get_param_values(cls, params, args, kwargs): ... + param_args = ... # type: Any + param_kwargs = ... # type: Any + task_id = ... # type: str + set_tracking_url = ... # type: Any + set_status_message = ... # type: Any + def __init__(self, *args, **kwargs) -> None: ... + def initialized(self): ... + @classmethod + def from_str_params(cls: Type[_T], params_str: Dict[str, str]) -> _T: ... + def to_str_params(self, only_significant: bool = ...): ... + def clone(self, cls: Optional[Any] = ..., **kwargs): ... + def __hash__(self): ... + def __eq__(self, other): ... + def complete(self) -> bool: ... + @classmethod + def bulk_complete(cls, parameter_tuples): ... + def output(self) -> _O: ... + def requires(self) -> _R: ... + def process_resources(self): ... + def input(self) -> _I: ... + def deps(self): ... + def run(self) -> None: ... + def on_failure(self, exception): ... + def on_success(self): ... + def no_unpicklable_properties(self): ... + +class MixinNaiveBulkComplete: + @classmethod + def bulk_complete(cls, parameter_tuples): ... + +class ExternalTask(Task[_R, _I, _O]): + run = ... # type: Any + +def externalize(taskclass_or_taskobject): ... + +class WrapperTask(Task[_R, _I, _O]): + def complete(self): ... + +class Config(Task[_R, _I, _O]): ... + +def getpaths(struct): ... +def flatten(struct: Union[Dict[Any, _O], Iterable[_O], _O]) -> List[_O]: ... +def flatten_output(task): ... diff --git a/stubs/luigi/tools/__init__.pyi b/stubs/luigi/tools/__init__.pyi new file mode 100644 index 0000000..d516037 --- /dev/null +++ b/stubs/luigi/tools/__init__.pyi @@ -0,0 +1,4 @@ +# Stubs for luigi.tools (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + diff --git a/stubs/luigi/tools/range.pyi b/stubs/luigi/tools/range.pyi new file mode 100644 index 0000000..ad0de09 --- /dev/null +++ b/stubs/luigi/tools/range.pyi @@ -0,0 +1,86 @@ +# Stubs for luigi.tools.range (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any +import luigi + +logger = ... # type: Any + +class RangeEvent(luigi.Event): + COMPLETE_COUNT = ... # type: str + COMPLETE_FRACTION = ... # type: str + DELAY = ... # type: str + +class RangeBase(luigi.WrapperTask): + of = ... # type: Any + of_params = ... # type: Any + start = ... # type: Any + stop = ... # type: Any + reverse = ... # type: Any + task_limit = ... # type: Any + now = ... # type: Any + param_name = ... # type: Any + @property + def of_cls(self): ... + def datetime_to_parameter(self, dt): ... + def parameter_to_datetime(self, p): ... + def datetime_to_parameters(self, dt): ... + def parameters_to_datetime(self, p): ... + def moving_start(self, now): ... + def moving_stop(self, now): ... + def finite_datetimes(self, finite_start, finite_stop): ... + def requires(self): ... + def missing_datetimes(self, finite_datetimes): ... + +class RangeDailyBase(RangeBase): + start = ... # type: Any + stop = ... # type: Any + days_back = ... # type: Any + days_forward = ... # type: Any + def datetime_to_parameter(self, dt): ... + def parameter_to_datetime(self, p): ... + def datetime_to_parameters(self, dt): ... + def parameters_to_datetime(self, p): ... + def moving_start(self, now): ... + def moving_stop(self, now): ... + def finite_datetimes(self, finite_start, finite_stop): ... + +class RangeHourlyBase(RangeBase): + start = ... # type: Any + stop = ... # type: Any + hours_back = ... # type: Any + hours_forward = ... # type: Any + def datetime_to_parameter(self, dt): ... + def parameter_to_datetime(self, p): ... + def datetime_to_parameters(self, dt): ... + def parameters_to_datetime(self, p): ... + def moving_start(self, now): ... + def moving_stop(self, now): ... + def finite_datetimes(self, finite_start, finite_stop): ... + +class RangeByMinutesBase(RangeBase): + start = ... # type: Any + stop = ... # type: Any + minutes_back = ... # type: Any + minutes_forward = ... # type: Any + minutes_interval = ... # type: Any + def datetime_to_parameter(self, dt): ... + def parameter_to_datetime(self, p): ... + def datetime_to_parameters(self, dt): ... + def parameters_to_datetime(self, p): ... + def moving_start(self, now): ... + def moving_stop(self, now): ... + def finite_datetimes(self, finite_start, finite_stop): ... + +def most_common(items): ... +def infer_bulk_complete_from_fs(datetimes, datetime_to_task, datetime_to_re): ... + +class RangeDaily(RangeDailyBase): + def missing_datetimes(self, finite_datetimes): ... + +class RangeHourly(RangeHourlyBase): + def missing_datetimes(self, finite_datetimes): ... + +class RangeByMinutes(RangeByMinutesBase): + def missing_datetimes(self, finite_datetimes): ... diff --git a/stubs/sqlalchemy/__init__.pyi b/stubs/sqlalchemy/__init__.pyi new file mode 100644 index 0000000..8d58e25 --- /dev/null +++ b/stubs/sqlalchemy/__init__.pyi @@ -0,0 +1,124 @@ +# Stubs for sqlalchemy (Python 2) + +from .sql import ( + alias, + and_, + asc, + between, + bindparam, + case, + cast, + collate, + column, + delete, + desc, + distinct, + except_, + except_all, + exists, + extract, + false, + func, + funcfilter, + insert, + intersect, + intersect_all, + join, + literal, + literal_column, + modifier, + not_, + null, + or_, + outerjoin, + outparam, + over, + select, + subquery, + table, + text, + true, + tuple_, + type_coerce, + union, + union_all, + update, +) + +from .types import ( + BIGINT, + BINARY, + BLOB, + BOOLEAN, + BigInteger, + Binary, + Boolean, + CHAR, + CLOB, + DATE, + DATETIME, + DECIMAL, + Date, + DateTime, + Enum, + FLOAT, + Float, + INT, + INTEGER, + Integer, + Interval, + LargeBinary, + NCHAR, + NVARCHAR, + NUMERIC, + Numeric, + PickleType, + REAL, + SMALLINT, + SmallInteger, + String, + TEXT, + TIME, + TIMESTAMP, + Text, + Time, + TypeDecorator, + Unicode, + UnicodeText, + VARBINARY, + VARCHAR, +) + +from .schema import ( + CheckConstraint, + Column, + ColumnDefault, + Constraint, + DefaultClause, + FetchedValue, + ForeignKey, + ForeignKeyConstraint, + Index, + MetaData, + PassiveDefault, + PrimaryKeyConstraint, + Sequence, + Table, + ThreadLocalMetaData, + UniqueConstraint, + DDL, +) + +from . import sql as sql +from . import schema as schema +from . import types as types +from . import exc as exc +from . import dialects as dialects +from . import pool as pool +# This should re-export orm but orm is totally broken right now +# from . import orm as orm + +from .inspection import inspect +from .engine import create_engine, engine_from_config + +__version__ = ... # type: int diff --git a/stubs/sqlalchemy/engine/__init__.pyi b/stubs/sqlalchemy/engine/__init__.pyi new file mode 100644 index 0000000..396cac5 --- /dev/null +++ b/stubs/sqlalchemy/engine/__init__.pyi @@ -0,0 +1,11 @@ +# Stubs for sqlalchemy.engine (Python 2) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from .base import Connection as Connection +from .base import Engine as Engine +from .base import Transaction as Transaction +from .result import RowProxy as RowProxy + +def create_engine(*args, **kwargs): ... +def engine_from_config(configuration, prefix=..., **kwargs): ... diff --git a/stubs/sqlalchemy/engine/base.pyi b/stubs/sqlalchemy/engine/base.pyi new file mode 100644 index 0000000..4f5ff43 --- /dev/null +++ b/stubs/sqlalchemy/engine/base.pyi @@ -0,0 +1,39 @@ +from typing import ( + Any, Generic, List, Optional, Tuple, Type, TypeVar +) +from types import TracebackType + +from .result import ResultProxy + +_T = TypeVar('_T') + +# new in 3.6 +class ContextManager(Generic[_T]): + def __enter__(self) -> _T: ... + def __exit__(self, exc_type: Optional[Type[BaseException]], + exc_val: Optional[Exception], + exc_tb: Optional[TracebackType]) -> bool: ... + + +# Dummy until I figure out something better. +class Connectable: + pass + +class Connection: + def begin(self) -> 'Transaction': ... + def execute(self, object: object, *multiparams: Any, **params: Any) -> ResultProxy: ... + def scalar(self, object: object, *multiparams: Any, **params: Any) -> Any: ... + +class Engine(object): + def execute(self, object: object, *multiparams: Any, **params: Any) -> ResultProxy: ... + def connect(self) -> Connection: ... + +class RowProxy: + def items(self) -> List[Tuple[Any, Any]]: ... + def keys(self) -> List[Any]: ... + def values(self) -> List[Any]: ... + def __getitem__(self, key: str): ... + +class Transaction(ContextManager): + def commit(self): ... + def rollback(self): ... diff --git a/stubs/sqlalchemy/engine/result.pyi b/stubs/sqlalchemy/engine/result.pyi new file mode 100644 index 0000000..a44553b --- /dev/null +++ b/stubs/sqlalchemy/engine/result.pyi @@ -0,0 +1,86 @@ +# Stubs for sqlalchemy.engine.result (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Iterable, List, Optional +from ..sql import expression as expression, sqltypes as sqltypes +# from ..sql import util as sql_util +# from sqlalchemy.cresultproxy import BaseRowProxy + +def rowproxy_reconstructor(cls, state): ... + +class BaseRowProxy(Iterable): + def __init__(self, parent, row, processors, keymap) -> None: ... + def __reduce__(self): ... + def values(self): ... + def __iter__(self): ... + def __len__(self): ... + def __getitem__(self, key): ... + def __getattr__(self, name): ... + +class RowProxy(BaseRowProxy): + def __contains__(self, key): ... + __hash__ = ... # type: Any + def __lt__(self, other): ... + def __le__(self, other): ... + def __ge__(self, other): ... + def __gt__(self, other): ... + def __eq__(self, other): ... + def __ne__(self, other): ... + def has_key(self, key): ... + def items(self): ... + def keys(self) -> List[str]: ... + def iterkeys(self): ... + def itervalues(self): ... + +class ResultMetaData: + case_sensitive = ... # type: Any + matched_on_name = ... # type: bool + def __init__(self, parent, cursor_description) -> None: ... + +class ResultProxy: + out_parameters = ... # type: Any + closed = ... # type: bool + context = ... # type: Any + dialect = ... # type: Any + cursor = ... # type: Any + connection = ... # type: Any + def __init__(self, context) -> None: ... + def keys(self) -> List[str]: ... + rowcount = ... # type: int + @property + def lastrowid(self): ... + @property + def returns_rows(self): ... + @property + def is_insert(self): ... + def close(self): ... + def __iter__(self): ... + def inserted_primary_key(self): ... + def last_updated_params(self): ... + def last_inserted_params(self): ... + @property + def returned_defaults(self): ... + def lastrow_has_defaults(self): ... + def postfetch_cols(self): ... + def prefetch_cols(self): ... + def supports_sane_rowcount(self): ... + def supports_sane_multi_rowcount(self): ... + def process_rows(self, rows): ... + def fetchall(self) -> List[RowProxy]: ... + def fetchmany(self, size: Optional[Any] = ...): ... + def fetchone(self) -> RowProxy: ... + def first(self): ... + def scalar(self): ... + +class BufferedRowResultProxy(ResultProxy): + size_growth = ... # type: Any + +class FullyBufferedResultProxy(ResultProxy): ... + +class BufferedColumnRow(RowProxy): + def __init__(self, parent, row, processors, keymap) -> None: ... + +class BufferedColumnResultProxy(ResultProxy): + def fetchall(self) -> List[RowProxy]: ... + def fetchmany(self, size: Optional[Any] = ...) -> List[RowProxy]: ... diff --git a/stubs/sqlalchemy/engine/url.pyi b/stubs/sqlalchemy/engine/url.pyi new file mode 100644 index 0000000..7880ccc --- /dev/null +++ b/stubs/sqlalchemy/engine/url.pyi @@ -0,0 +1,27 @@ +# Stubs for sqlalchemy.engine.url (Python 2) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, AnyStr +from .. import dialects + +# registry = dialects.registry + +class URL: + drivername = ... # type: Any + username = ... # type: Any + password = ... # type: Any + host = ... # type: Any + port = ... # type: Any + database = ... # type: Any + query = ... # type: Any + def __init__(self, drivername, username=..., password=..., host=..., port=..., database=..., query=...) -> None: ... + def __to_string__(self, hide_password=...): ... + def __hash__(self): ... + def __eq__(self, other): ... + def get_backend_name(self): ... + def get_driver_name(self): ... + def get_dialect(self): ... + def translate_connect_args(self, names=..., **kw): ... + +def make_url(name_or_url: AnyStr) -> URL: ... diff --git a/stubs/sqlalchemy/exc.pyi b/stubs/sqlalchemy/exc.pyi new file mode 100644 index 0000000..ef3c7af --- /dev/null +++ b/stubs/sqlalchemy/exc.pyi @@ -0,0 +1,27 @@ +# Stubs for sqlalchemy.exc (Python 3.5) +# +# NOTE: This dynamically typed is an excerpt from a +# stub automatically generated by stubgen. + +from typing import Any, Optional + +class SQLAlchemyError(Exception): ... + +class StatementError(SQLAlchemyError): + statement = ... # type: Any + params = ... # type: Any + orig = ... # type: Any + detail = ... # type: Any + def __init__(self, message, statement, params, orig) -> None: ... + def add_detail(self, msg): ... + def __reduce__(self): ... + def __unicode__(self): ... + +class DBAPIError(StatementError): + @classmethod + def instance(cls, statement, params, orig, dbapi_base_err, connection_invalidated: bool = ..., dialect: Optional[Any] = ...): ... + def __reduce__(self): ... + connection_invalidated = ... # type: Any + def __init__(self, statement, params, orig, connection_invalidated: bool = ...) -> None: ... + +class DatabaseError(DBAPIError): ... diff --git a/stubs/sqlalchemy/sql/__init__.pyi b/stubs/sqlalchemy/sql/__init__.pyi new file mode 100644 index 0000000..e69de29 diff --git a/stubs/sqlalchemy/sql/elements.pyi b/stubs/sqlalchemy/sql/elements.pyi new file mode 100644 index 0000000..eabe5b3 --- /dev/null +++ b/stubs/sqlalchemy/sql/elements.pyi @@ -0,0 +1,25 @@ +from typing import Any, Optional + +class Visitable(object): + pass + +class ClauseElement(Visitable): + __visit_name__ = ... # type: str + supports_execution = ... # type: bool + bind = ... # type: Any + is_selectable = ... # type: bool + is_clause_element = ... # type: bool + description = ... # type: Any + def unique_params(self, *optionaldict, **kwargs): ... + def params(self, *optionaldict, **kwargs): ... + def compare(self, other, **kw): ... + def get_children(self, **kwargs): ... + def self_group(self, against: Optional[Any] = ...): ... + # @util.dependencies("sqlalchemy.engine.default") + # def compile(self, default, bind: Optional[Any] = ..., dialect: Optional[Any] = ..., **kw): ... + def compile(self, bind: Optional[Any] = ..., dialect: Optional[Any] = ..., **kw): ... + def __and__(self, other): ... + def __or__(self, other): ... + def __invert__(self): ... + def __bool__(self): ... + __nonzero__ = ... # type: Any diff --git a/stubs/sqlalchemy/sql/expression.pyi b/stubs/sqlalchemy/sql/expression.pyi new file mode 100644 index 0000000..f361bd2 --- /dev/null +++ b/stubs/sqlalchemy/sql/expression.pyi @@ -0,0 +1,7 @@ +from .elements import ClauseElement + +class Select(ClauseElement): + @property + def columns(self): ... + c = ... # type: Any + def where(self, whereclause): ... From f08bd2bf23244e8a9210d1a07ec57a840acc1461 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 22 Jan 2018 11:21:49 -0600 Subject: [PATCH 314/507] import heron_tasks luigi support https://github.com/kumc-bmi/heron/tree/para_etl_3985/heron_tasks 21fa46c on Dec 13, 2017 --- CONTRIBUTING.md | 132 ++++ Oracle/epic_facts_load.sql | 92 +++ Oracle/epic_flowsheets_transform.sql | 577 +++++++++++++++++ Oracle/etl_tests_init.sql | 84 +++ Oracle/migrate_fact_upload.sql | 22 + README.md | 56 +- client.cfg | 58 ++ epic_flowsheets.py | 192 ++++++ etl_tasks.py | 897 +++++++++++++++++++++++++++ eventlog.py | 152 +++++ logging.cfg | 81 +++ param_val.py | 37 ++ pythonjsonlogger/__init__.py | 0 pythonjsonlogger/jsonlogger.py | 141 +++++ requirements.txt | 53 ++ script_lib.py | 306 +++++++++ setup.cfg | 49 ++ sql-style.xml | 453 ++++++++++++++ sql_syntax.py | 238 +++++++ stubs/cx_Oracle.pyi | 429 +++++++++++++ stubs/luigi/__init__.pyi | 20 + stubs/luigi/configuration.pyi | 26 + stubs/luigi/contrib/__init__.pyi | 4 + stubs/luigi/contrib/sqla.pyi | 43 ++ stubs/luigi/event.pyi | 15 + stubs/luigi/interface.pyi | 37 ++ stubs/luigi/local_target.pyi | 35 ++ stubs/luigi/parameter.pyi | 131 ++++ stubs/luigi/rpc.pyi | 26 + stubs/luigi/target.pyi | 47 ++ stubs/luigi/task.pyi | 107 ++++ stubs/luigi/tools/__init__.pyi | 4 + stubs/luigi/tools/range.pyi | 86 +++ stubs/sqlalchemy/__init__.pyi | 124 ++++ stubs/sqlalchemy/engine/__init__.pyi | 11 + stubs/sqlalchemy/engine/base.pyi | 39 ++ stubs/sqlalchemy/engine/result.pyi | 86 +++ stubs/sqlalchemy/engine/url.pyi | 27 + stubs/sqlalchemy/exc.pyi | 27 + stubs/sqlalchemy/sql/__init__.pyi | 0 stubs/sqlalchemy/sql/elements.pyi | 25 + stubs/sqlalchemy/sql/expression.pyi | 7 + 42 files changed, 4974 insertions(+), 2 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 Oracle/epic_facts_load.sql create mode 100644 Oracle/epic_flowsheets_transform.sql create mode 100644 Oracle/etl_tests_init.sql create mode 100644 Oracle/migrate_fact_upload.sql create mode 100644 client.cfg create mode 100644 epic_flowsheets.py create mode 100644 etl_tasks.py create mode 100644 eventlog.py create mode 100644 logging.cfg create mode 100644 param_val.py create mode 100644 pythonjsonlogger/__init__.py create mode 100644 pythonjsonlogger/jsonlogger.py create mode 100644 requirements.txt create mode 100644 script_lib.py create mode 100644 setup.cfg create mode 100644 sql-style.xml create mode 100644 sql_syntax.py create mode 100644 stubs/cx_Oracle.pyi create mode 100644 stubs/luigi/__init__.pyi create mode 100644 stubs/luigi/configuration.pyi create mode 100644 stubs/luigi/contrib/__init__.pyi create mode 100644 stubs/luigi/contrib/sqla.pyi create mode 100644 stubs/luigi/event.pyi create mode 100644 stubs/luigi/interface.pyi create mode 100644 stubs/luigi/local_target.pyi create mode 100644 stubs/luigi/parameter.pyi create mode 100644 stubs/luigi/rpc.pyi create mode 100644 stubs/luigi/target.pyi create mode 100644 stubs/luigi/task.pyi create mode 100644 stubs/luigi/tools/__init__.pyi create mode 100644 stubs/luigi/tools/range.pyi create mode 100644 stubs/sqlalchemy/__init__.pyi create mode 100644 stubs/sqlalchemy/engine/__init__.pyi create mode 100644 stubs/sqlalchemy/engine/base.pyi create mode 100644 stubs/sqlalchemy/engine/result.pyi create mode 100644 stubs/sqlalchemy/engine/url.pyi create mode 100644 stubs/sqlalchemy/exc.pyi create mode 100644 stubs/sqlalchemy/sql/__init__.pyi create mode 100644 stubs/sqlalchemy/sql/elements.pyi create mode 100644 stubs/sqlalchemy/sql/expression.pyi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a8e9f6b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,132 @@ +## Design: ETL Tasks and SQL Scripts + +The main tasks are in: + + - *epic_flowsheets* -- ETL tasks for Epic Flowsheets to i2b2 + +which is based on a design in: + + - *etl_tasks* -- Source-agnostic Luigi ETL Task support + - *script_lib* -- library of SQL scripts + - *sql_syntax* -- break SQL scripts into statements, etc. + +Tasks such as `epic_flowsheets.FlowsheetViews` are based on SQL +scripts such as `sql_scripts/epic_flowsheets_transform.sql` wrapped in +a `etl_tasks.SqlScriptTask`. + + +### SQL Script Library Design, Style and Conventions + +Each script should start with a header comment and some +dependency-checking queries. See `script_lib.py` for details. + +SQL should be written in lowercase, indented 2 spaces, 120 maximum line +length. More details are in the evolving `sql-style.xml` sqldeveloper +style profile. + + - *ISSUE*: sqldeveloper 3 vs. 4 style files? + +See also notes on value enumerations in the header of `sql_scripts/cms_keys.pls`. +*TODO*: port those notes from GROUSE. + + +## Python doctest for story telling and unit testing + +Each python module header should tell a story using [doctest][], +i.e. examples that are also unit tests. + +You can run them one module at a time: + + (luigi)$ python -m doctest script_lib.py -v + Trying: + Script.migrate_fact_upload.title + Expecting: + 'append data from a workspace table.' + ... + 22 tests in 13 items. + 22 passed and 0 failed. + Test passed. + +Or install [nose][] and run all modules at once: + + (grouse-etl)% nosetests --with-doctest + ...................... + ---------------------------------------------------------------------- + Ran 22 tests in 0.658s + + OK + +[doctest]: http://docs.python.org/2/library/doctest.html +[nose]: https://pypi.python.org/pypi/nose/ + + +## Python code style + +We appreciate object capability discipline and the "don't call us, +we'll call you" style that facilitates unit testing with mocks. + + - *ISSUE*: Luigi's design doesn't seem to support this idiom. + Constructors are implict and tasks parameters have to be + serializable, which works against the usual closure + object pattern. Also, the task cache is global mutable + state. + +We avoid mutable state, preferring functional style. + + - *NOTE*: PEP8 tools warn against assinging a lambda to a name, + suggesting `def` instead. We're fine with it; hence + `ignore = E731` in `setup.cfg`. + + +We follow PEP8. The first line of a module or function docstring +should be a short description; if it is blank, either the item is in +an early stage of development or the name and doctests are supposed to +make the purpose obvious. + + - *NOTE*: with static type annotations, the 79 character line + length limit is awkward; hence we use 99 in `setup.cfg`. + + - *ISSUE*: Dan didn't realize until recently that PEP8 recommends + triple double quotes over triple single quotes for + docstrings. He's in the habit of using single quotes + to minimize use of the shift key. + + +## Checking the code + +Once dependencies in `requirements.txt` are satisfied, code should +pass tests, style checks, and static type checking: + + $ nosetests && flake8 . && mypy . + +_tested with mypy-0.521_ + +### Checking in emacs + +To check with `M-x compile` in emacs, first use `M-x pyvenv-activate` +from the [pyvenv][] package. + +To check continuously as you edit, use [flycheck][] and activate +likewise. + +[pyvenv]: https://melpa.org/#/pyvenv +[pyvent]: https://melpa.org/#/flycheck + + +## Luigi Troubleshooting + +**ISSUE**: why won't luigi find modules in the current directory? + Use `PYTHONPATH=. luigi ...` if necessary. + +Most diagnostics are self-explanatory; `etl_tasks` includes +`SQLScriptError` and `ConnectionProblem` exception classes intended to +improve diagnostics + +One challenging diagnostic is: + + RuntimeError: Unfulfilled dependency at run time: DiagnosesLoad_oracle___dconnol_CMS_DEID_SAMPLE_1438246788671_bd6231c982 + +It seems to indicate that the `.complete()` test on a required task +fails even after that task has been `.run()`. For example, the `select +count(*)` completion test in a load script might have failed because +of incorrect join constraints. diff --git a/Oracle/epic_facts_load.sql b/Oracle/epic_facts_load.sql new file mode 100644 index 0000000..6bff120 --- /dev/null +++ b/Oracle/epic_facts_load.sql @@ -0,0 +1,92 @@ +/** epic_facts_load.sql - Load observations from Epic Clarity. + +Copyright (c) 2012-2017 University of Kansas Medical Center +part of the HERON* open source codebase; see NOTICE file for license details. +* http://informatics.kumc.edu/work/wiki/HERON + +see also epic_dimensions_load.sql, + http://informatics.kumc.edu/work/wiki/HeronLoad +*/ + +/* Check for id repository, uploader service tables */ +select * from NightHeronData.observation_fact where 1 = 0; +select * from NightHeronData.upload_status where 1 = 0; + +/* We're wasting our time if an upload_status record isn't in place. */ +select case count(*) when 1 then 1 else 1/0 end as upload_status_exists from ( +select * from NightHeronData.upload_status up +where up.upload_id = :upload_id +); + + +create table observation_fact_&&upload_id as +select * from NightHeronData.observation_fact where 1=0; + +insert /*+ append */ into observation_fact_&&upload_id ( + patient_num, encounter_num, sub_encounter, + concept_cd, + provider_id, + start_date, + modifier_cd, + instance_num, + valtype_cd, + tval_char, + nval_num, + valueflag_cd, + units_cd, + end_date, + location_cd, + update_date, + import_date, upload_id, download_date, sourcesystem_cd) +select pmap.patient_num, + coalesce(emap.encounter_num, -abs(ora_hash(to_char(f.start_date, 'YYYYMMDD') || f.patient_ide))), -- TODO: pat_day func + f.encounter_ide, + f.concept_cd, + f.provider_id, + f.start_date, + f.modifier_cd, + f.instance_num, + f.valtype_cd, + f.tval_char, + f.nval_num, + f.valueflag_cd, + f.units_cd, + f.end_date, + f.location_cd, + f.update_date, + sysdate, up.upload_id, :download_date, up.source_cd +from &&epic_fact_view f + join NightHeronData.patient_mapping pmap + on pmap.patient_ide = f.patient_ide + and pmap.patient_ide_source = :pat_source_cd + and pmap.patient_ide_status = 'A' -- TODO: (select active from i2b2_status) + left join NightHeronData.encounter_mapping emap + on emap.encounter_ide = f.encounter_ide + and emap.encounter_ide_source = :enc_source_cd + and emap.encounter_ide_status = 'A' + , NightHeronData.upload_status up +where up.upload_id = :upload_id + and f.patient_ide between :pat_id_lo and :pat_id_hi + &&log_fact_exceptions +; +commit +; + + +/* For this upload of data, check primary key constraints. +This could perhaps be factored out of all the load scripts into i2b2_facts_deid.sql, +at the cost of slowing down the select count(*) for summary stats, below. + +TODO: check key constraint + +alter table observation_fact_&&upload_id + enable constraint observation_fact_pk + -- TODO: log errors ... ? #2117 + ; + */ + + +select count(*) +from NightHeronData.upload_status +where transform_name = :task_id and load_status = 'OK'; + diff --git a/Oracle/epic_flowsheets_transform.sql b/Oracle/epic_flowsheets_transform.sql new file mode 100644 index 0000000..fc40305 --- /dev/null +++ b/Oracle/epic_flowsheets_transform.sql @@ -0,0 +1,577 @@ +/** epic_flowsheets_transform.sql -- prepare to load Epic EMR flowsheet facts + +Copyright (c) 2012 University of Kansas Medical Center +part of the HERON* open source codebase; see NOTICE file for license details. +* http://informatics.kumc.edu/work/wiki/HERON + + * See also epic_flowsheets_multiselect.sql, epic_etl.py, epic_facts_load.sql + * + * References: + * + * Lesson 4: Flowsheet Data + * Clarity Data Model - EpicCare Inpatient Spring 2008 Training Companion + * https://userweb.epic.com/epiclib/epicdoc/EpicWiseSpr08/Clarity/Clarity%20Training%20Companions/CLR209%20Data%20Model%20-%20EpicCare%20Inpatient/04TC%20Flowsheet%20Data.doc + */ + +select test_name from etl_tests where 'dep' = 'etl_tests_init.sql'; + +-- Check that this is the ID instance +select flo_meas_id from CLARITY.ip_flwsht_meas where 1=0; + +-- to see the time detail... +alter session set NLS_DATE_FORMAT = 'YYYY-MM-DD HH:MI'; + +-- define heron_obs_sample = sample (0.001, 1234) + +create or replace view etl_test_domain_flowsheets as +select 'Flowsheets' test_domain from dual; + +insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) +with test_key as ( + select test_domain, + 'val_type_seen_before' test_name from etl_test_domain_flowsheets + ) +, test_values as ( + select count(*) test_value, test_key.* from ( + select distinct zcvt.name + from CLARITY.ip_flwsht_meas ifm + join CLARITY.ip_flo_gp_data ifgd + on ifm.flo_meas_id= ifgd.flo_meas_id + join CLARITY.zc_val_type zcvt on ifgd.val_type_c = zcvt.val_type_c + where zcvt.name not in ( + 'Blood Pressure', 'Category Type', 'Concentration', 'Custom List', + 'Date', 'Dose', 'Height', 'Numeric Type', 'Patient Height', + 'Patient Weight', 'Rate', 'String Type', 'Temperature', 'Time', 'Weight') + ), test_key + ) +select test_value, test_domain, test_name, sq_result_id.nextval, sysdate +from test_values +; + + +/*************** + * flo_meas_type -- per-flo_meas_id transformation + +Starting with relevant columns from ip_flo_gp_data, +determine i2b2 valtype_cd, UNITS_CD. + +See also i2b2_facts_deid.sql regarding identified/de-identified data +flag in valtype_cd. + +Values for height, weight, and temperature are (for reasons lost to history) +converted to SI units. + +Also, trivially set up MODIFIER_CD, CONFIDENCE_NUM. + +TODO: consider migrating to CSV spreadsheet. Too bad SQL doesn't + have relation constants a la R read.table() or JSON. + + */ +-- compute i2b2 valtype_cd for each flo_meas_id +create or replace view flo_meas_type as +select ifgd.flo_meas_id, ifgd.disp_name + , ifgd.multi_sel_yn + , ifgd.record_state_c + , zcvt.name val_type + , case + when zcvt.name in ( + 'Numeric Type', + 'Blood Pressure', 'Temperature', 'Time', + 'Patient Weight', 'Weight', + 'Patient Height', 'Height') then 'N' + when zcvt.name in ('Custom List', 'Category Type') then '@' + -- only add ID information to nightheron + when zcvt.name in ('String Type') then 'Ti' + when zcvt.name in ('Date') then 'D' + else null -- val_type_seen_before test keeps these bounded + end valtype_cd + , 'KUH|FLO_MEAS_ID:' || ifgd.flo_meas_id CONCEPT_CD + + -- Convert to SI units per 2011 design. + , case + when zcvt.name in ( + /* Epic documentation doesn't give units for 'Weight', + so we've determined emperically that it's oz. + See also mean Birth Weight test based on flo_stats_num + in epic_flowsheets_multiselect.sql */ + 'Patient Weight', 'Weight') then 1.0 / 16.0 / 2.2 -- 16 oz/lb; 2.2kg/lb + when zcvt.name in ( + 'Patient Height', 'Height') then 2.54 -- 2.54 cm/in + when zcvt.name in ( + 'Temperature') then 5/9 -- F to C + else 1 + end scale + , case + when zcvt.name in ( + 'Temperature') then -32 -- F to C + else 0 + end bias + , case + when zcvt.name in ( + 'Patient Weight', 'Weight') then 'kg' + when zcvt.name in ( + 'Temperature') then 'C' + when zcvt.name in ( + 'Time') then 's' + when zcvt.name in ( + 'Patient Height', 'Height') then 'cm' + when zcvt.name in ( + 'Blood Pressure') then 'mmHg' + else ifgd.units + end UNITS_CD + , '@' MODIFIER_CD + , to_number(null) CONFIDENCE_NUM +from CLARITY.ip_flo_gp_data ifgd +left join CLARITY.zc_val_type zcvt on ifgd.val_type_c = zcvt.val_type_c +; + + +/*************** + * flowsheet_day -- per-patient-day transformation + +Starting with ip_flwsht_rec, compute ENCOUNTER_IDE, PATIENT_IDE +(to be mixed with patient_mapping and encounter_mapping in epic_facts_load.sql). + +Trivially set LOCATION_CD. + + */ +create or replace view flowsheet_day as +select ifr.record_date + , ifr.pat_id + , ifr.fsd_id + , ifr.INPATIENT_DATA_ID + -- i2b2 equivalents common to many/all datatypes + + -- patient day; see patient_day_visit view + , TO_CHAR(ifr.record_date,'YYYYMMDD') || ifr.pat_id + ENCOUNTER_IDE + , ifr.pat_id PATIENT_IDE + , '@' LOCATION_CD +from CLARITY.ip_flwsht_rec ifr +; + + +/*************** + * flowsheet_obs -- per-measurement transformation common to all datatypes + +Starting with ip_flwsht_meas, compute instance_num, +start_date, end_date, update_date. + +Filter out rows where meas_value is null. + +Trivially (for now) set PROVIDER_ID, valueflag_cd. + + */ +create or replace view flowsheet_obs as +select /*+ parallel(16) */ -- ISSUE: parameterize parallel_degree? + ifm.fsd_id + , ifm.line + , ifm.flo_meas_id + , ifm.meas_value + , ifm.recorded_time + , ifm.entry_time + -- per-measurement conversion to i2b2 terms + , '@' PROVIDER_ID -- todo: use ifm.TAKEN_USER_ID + , recorded_time START_DATE + , ifm.fsd_id * 100000 + ifm.line instance_num + , '@' valueflag_cd -- TODO #3850: [H]igh/[L]ow/[A]bnormal based on ifm.ABNORMAL_C + , recorded_time END_DATE + , entry_time UPDATE_DATE +from CLARITY.ip_flwsht_meas ifm +where meas_value is not null +; + + +/******************** + * flowsheet_num_data - handle number syntax + +Starting with flowsheet_obs, split diastolic and systolic into separate rows, +flag (but don't filter) illegal numerals, and convert the rest to_number(). + +Values such as 2.22222222222222E+22 don't fit in nval_num, +which is declared NUMBER(18,5). Let's throw out values using E notation. +The `numerals_filtered` test (in epic_flowsheets_multiselect) verifies that +we're not throwing away more than a handful. + +Stats in epic_flowsheets_multiselect.sql are based on this view, +before we join with per-patient-day or per-flow-measure info. + + */ +create or replace view flowsheet_num_data as +with +parts as ( + select flo_meas_id, bp_part, pattern + from flo_meas_type fmt + left join ( + select 'Blood Pressure' val_type, '_DIASTOLIC' bp_part, '\d+$' pattern from dual +union all select 'Blood Pressure' val_type, '_SYSTOLIC' bp_part, '^\d+' pattern from dual + ) parts on parts.val_type = fmt.val_type + where fmt.valtype_cd like 'N%' +) +, numerals as ( + select fsd_id, line + , obs.flo_meas_id + , bp_part + , meas_value + , recorded_time + , entry_time + , case + when parts.pattern is not null + then regexp_substr(meas_value, parts.pattern) + else meas_value + end numeral + , START_DATE + , END_DATE + , instance_num + , UPDATE_DATE + , PROVIDER_ID + , valueflag_cd + from flowsheet_obs obs + join parts on parts.flo_meas_id = obs.flo_meas_id +) +, num_check as ( + select numerals.* + , case + when numeral is null then null + when regexp_like(numeral, '^-?\d*(\.\d+)?$') then 1 + else 0 + end ok + from numerals +) + select num_check.* + , case + when ok = 1 then to_number(numeral) + else null + end meas_value_num + from num_check +; + + +/************** + * numerictypeflows - Numeric, Blood Pressure, Weight, etc. + +Build concept_cd using blood pressure suffix; apply units bias and scale; +trivially set TVAL_CHAR. + +Join flowsheet_num_data with flo_meas_type and flowsheet_day to fill in +remaining observation_fact style fields. + + */ +create or replace view numerictypeflows as +select ENCOUNTER_IDE + , PATIENT_IDE + , 'KUH|FLO_MEAS_ID:' || num_data.flo_meas_id || bp_part concept_cd + , PROVIDER_ID + , START_DATE + , MODIFIER_CD + , instance_num + , valtype_cd + , 'E' TVAL_CHAR -- i2b2 doc says 'EQ' but demo data says 'E' + , case when fmt.bias <> 0 or fmt.scale <> 1 + then + -- Round so as not to imply more precision than was measured. + round((meas_value_num + fmt.bias) * fmt.scale, 3) + else + meas_value_num + end nval_num + , valueflag_cd + , UNITS_CD + , END_DATE + , LOCATION_CD + , CONFIDENCE_NUM + , UPDATE_DATE +from flowsheet_num_data num_data +join flo_meas_type fmt on fmt.flo_meas_id = num_data.flo_meas_id +join flowsheet_day fsd on fsd.fsd_id = num_data.fsd_id +where num_data.ok = 1 +; + +-- select * from numerictypeflows; + + +/*********** + * datemeasureflows - for value_type_name "Date" + * + * STRANGE! Syntax is EITHER: + * - a number of days since Dec 31, 1840 or + * - a date in 'MM/DD/YYYY' format + */ +create or replace view datemeasureflows as +Select + ENCOUNTER_IDE + , PATIENT_IDE + , CONCEPT_CD + , PROVIDER_ID + , START_DATE + , MODIFIER_CD + , instance_num + , 'D' VALTYPE_CD, + case + when substr(meas_value , 1, instr(meas_value , '/') -1 ) is NULL + then to_char(to_date('12/31/1840', 'MM/DD/YYYY') + meas_value, 'YYYY-MM-DD') + else to_char(to_date(meas_value, 'MM/DD/YYYY'), 'YYYY-MM-DD') + end TVAL_CHAR + , null NVAL_NUM + , valueflag_cd + , UNITS_CD + , END_DATE + , LOCATION_CD + , CONFIDENCE_NUM + , UPDATE_DATE +from flowsheet_obs obs +join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id +join flowsheet_day fsd on fsd.fsd_id = obs.fsd_id +where valtype_cd like 'D%'; + + +/* todo:: test date-shifting when VALUE_TYPE_NAME='Date' + e.g. 11568 or 700 PLACEMENT DATE (#299) */ + + +/***************** + * idstringtypeflows -- free text + +For measures identifed (in flo_meas_type) as free text, set tval_char. + +Trivially set nval_num. + */ +create or replace view idstringtypeflows as +select + fsd.ENCOUNTER_IDE + , fsd.PATIENT_IDE + , fmt.CONCEPT_CD + , obs.PROVIDER_ID + , obs.START_DATE + , fmt.MODIFIER_CD + , obs.instance_num + , fmt.VALTYPE_CD + , obs.meas_value TVAL_CHAR + , to_number(null) NVAL_NUM + , obs.valueflag_cd + , fmt.UNITS_CD + , obs.END_DATE + , fsd.LOCATION_CD + , fmt.CONFIDENCE_NUM + , obs.UPDATE_DATE +from flowsheet_obs obs +join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id +join flowsheet_day fsd on fsd.fsd_id = obs.fsd_id +where valtype_cd like 'T%'; + + +/***************** + * deidstringtypeflows -- deid-ed free text + +For measures identifed (in flo_meas_type) as free text, and have corresponding +de-identified meas_values. + +As an intermediate step create `approved_deid_flowsheets` containing all of the +de-identified free text notes that have been approved for inclusion. + +Trivially set nval_num. + */ + + +whenever sqlerror continue; +drop table approved_deid_flowsheets; +whenever sqlerror exit; + + +create table approved_deid_flowsheets ( + fsd_id number + , line number + , flo_meas_id number + , meas_value varchar(4000) + , primary key (fsd_id, line)); + + +whenever sqlerror continue; +drop table approved_flo_meas_ids; +-- Oracle gave really bad performance while `deidentify.deid_flowsheet_approval` +-- was a view. +create table approved_flo_meas_ids as +select * from deidentify.deid_flowsheet_approval where approved_to_deid = 1; + +insert into approved_deid_flowsheets +(fsd_id, line, flo_meas_id, meas_value) +select fsd_id, line, flo_meas_id, meas_value +from ( + -- `fsd_id` and `line` are extracted from `flo_notes.id` + -- see: heron_staging/deidentify/Flowsheets-DEID-Stage.sql + select to_number(substr(to_char(id), + 1, + length(to_char(id))-5)) as fsd_id + , to_number(substr(id, -5)) as line + , to_number(note_id) as flo_meas_id + , deid_note_text as meas_value + from deidentify.flo_notes) deid + -- Include only meas_values that have flo_meas_id approved. +where exists (select * from approved_flo_meas_ids af + where deid.flo_meas_id = af.flo_meas_id + ); +whenever sqlerror exit; + + +insert into etl_test_values (test_value, test_domain, test_name, result_id, + result_date) +with test_key as ( + select test_domain, + 'staged_deid_flowsheet' test_name from etl_test_domain_flowsheets + ) +, test_values as ( + select count(*) test_value, test_key.* from approved_deid_flowsheets, test_key + ) +select test_value, test_domain, test_name, sq_result_id.nextval, sysdate +from test_values +; + +-- Did `staged_deid_flowsheet` fail? First check to see if the necessary +-- staged tables exist. + +-- select * from deidentify.flo_notes; +-- select * from deidentify.deid_flowsheet_approval; + +create or replace view deidstringtypeflows as +select + fsd.ENCOUNTER_IDE + , fsd.PATIENT_IDE + , fmt.CONCEPT_CD + , obs.PROVIDER_ID + , obs.START_DATE + , fmt.MODIFIER_CD + , obs.instance_num + , 'Td' as valtype_cd + , deid.meas_value TVAL_CHAR + , to_number(null) NVAL_NUM + , obs.valueflag_cd + , fmt.UNITS_CD + , obs.END_DATE + , fsd.LOCATION_CD + , fmt.CONFIDENCE_NUM + , obs.UPDATE_DATE +from flowsheet_obs obs +join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id +join flowsheet_day fsd on fsd.fsd_id = obs.fsd_id +join approved_deid_flowsheets deid + on deid.fsd_id = obs.fsd_id + and deid.line = obs.line +where valtype_cd like 'T%'; + + +/******************** + * flowsheet_nom_data - nominal data, including multi select + +Stats in epic_flowsheets_multiselect.sql are based on this view, +before we join with per-patient-day or per-flow-measure info. + +The flow_measure_multi table (eventually) maps 'x;y;z' multi-select values +to the constituent 'x', 'y', and 'z' values. Create the table so we can +refer to it, but leave populating it to epic_flowsheets_multiselect.sql. + + */ +whenever sqlerror continue; +drop table flow_measure_multi; +whenever sqlerror exit; +create table flow_measure_multi as +select fm.meas_value + , 0 val_ix + , fm.meas_value choice_label +from flowsheet_obs fm +where 1=0; + +create or replace view flowsheet_nom_data as + select fsd_id, obs.line + , obs.flo_meas_id + , fmt.multi_sel_yn + , obs.meas_value + , fmm.val_ix + , fmm.choice_label multi_choice_label + , coalesce(fmm.choice_label, obs.meas_value) choice_label + , recorded_time + , entry_time + , START_DATE + , END_DATE + , UPDATE_DATE + , PROVIDER_ID + , valueflag_cd + from flowsheet_obs obs + join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id + and fmt.valtype_cd = '@' -- valtype_cd for nominal data + left join flow_measure_multi fmm + -- Try not to look in flow_measure_multi unless + -- except when we have multi_sel_yn and a ';' . + on fmm.meas_value = case + when fmt.multi_sel_yn = '1' + and obs.meas_value like '%;%' + then obs.meas_value + else null + end +; + +-- "forward reference" flo_stats_nom table for use in flowsheets_concepts.sql +whenever sqlerror continue; +drop table flo_stats_nom; +whenever sqlerror exit; + +create table flo_stats_nom as +select flo_meas_id, meas_value choice_label + , 0 qty + , 0 tot + , 99.9 pct + , date '2001-01-01' recorded_time_min + , date '2001-01-01' recorded_time_max +from flowsheet_nom_data +where 1 = 0 +; + + +/************* + * selectflows -- Custom List nominal observations + +Compute concept_cd using flo_meas_id and ip_flo_cust_list.line where available; +fall back to hash of choice label. + +Trivially set tval_char, nval_num. + */ + +create or replace view selectflows as +Select ENCOUNTER_IDE + , PATIENT_IDE + , (case when ifcl.line is not null + then 'KUH|FLO_MEAS_ID+LINE:' || obs.flo_meas_id || '_' || ifcl.line + else 'KUH|FLO_MEAS_ID+hash:' || obs.flo_meas_id || '_' || ora_hash(obs.choice_label) + end) CONCEPT_CD + , PROVIDER_ID + , START_DATE + , MODIFIER_CD + -- Handle cases such as 'Sinus Tachycardia;ST' + -- where multiple choices map to the same ifcl.line + -- by including the val_ix (1, 2) in instance_num. + , (obs.fsd_id * 100000 + obs.line) * 10 + coalesce(val_ix, 0) instance_num + , VALTYPE_CD + , '@' TVAL_CHAR + , to_number(null) NVAL_NUM + , valueflag_cd + , UNITS_CD + , END_DATE + , LOCATION_CD + , CONFIDENCE_NUM + , UPDATE_DATE +from flowsheet_nom_data obs +join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id +join flowsheet_day fsd on fsd.fsd_id = obs.fsd_id +left join CLARITY.ip_flo_cust_list ifcl + on ifcl.flo_meas_id = obs.flo_meas_id + and (obs.choice_label = ifcl.abbreviation or + obs.choice_label = ifcl.custom_value) +where valtype_cd = '@'; + +-- select * from multiselectflows where rownum < 200; + + +/* complete check */ +create or replace view epic_flowsheets_txform_sql as +select &&design_digest design_digest from dual; + +select 1 up_to_date +from epic_flowsheets_txform_sql where design_digest = &&design_digest; diff --git a/Oracle/etl_tests_init.sql b/Oracle/etl_tests_init.sql new file mode 100644 index 0000000..21db271 --- /dev/null +++ b/Oracle/etl_tests_init.sql @@ -0,0 +1,84 @@ +/* etl_tests_init -- Initialize ETL tests + +Copyright (c) 2015 University of Kansas Medical Center +part of the HERON* open source codebase; see NOTICE file for license details. +* http://informatics.kumc.edu/work/wiki/HERON + +References: + * KUMC Informatics ticket #2781 +*/ + +whenever sqlerror continue; +drop table etl_tests; +whenever sqlerror exit; + +create table etl_tests ( + /* Description of test */ + test_domain varchar(256) not null -- Such as top-level folder in HERON that's affected (Frontiers Registry...) + , test_name varchar(1024) not null -- User friendly name (hictr_mrn_leading_zero...) + /* Pass/Fail criteria - can define one or both of the following - INCLUSIVE */ + , lower_bound number null + , upper_bound number null + , units varchar(256) null -- What are we counting? (patients, facts, concepts, etc.) + , test_description varchar(4000) null -- More detailed description - how is the end-user affected if this test fails? + ); + +alter table etl_tests add constraint pk_etl_tests primary key (test_domain, test_name); + +-- Table to store test values (inserted during ETL) +whenever sqlerror continue; +drop table etl_test_values; +whenever sqlerror exit; + +create table etl_test_values as + select test_domain, test_name from etl_tests where 1=0; + +alter table etl_test_values + add (test_value number not null + , detail_char_1 varchar(256) + , detail_char_2 varchar(256) + , detail_num_1 number + , detail_num_2 number + , test_note varchar(256) + , result_id number not null + , result_date date not null); + +alter table etl_test_values add constraint pk_etl_test_results primary key (result_id); + +-- I wish we had "auto increment". We can use a trigger but then we need PL/SQL. +whenever sqlerror continue; +drop sequence sq_result_id; +whenever sqlerror exit; +create sequence sq_result_id; + +/* Insert some example tests for debugging. */ +-- Example test that doesn't have something in the CSV +insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) +with test_key as ( + select 'test' test_domain, + 'test_not_found_in_csv' test_name from dual + ) +select 666 test_value, test_key.*, sq_result_id.nextval, sysdate from test_key; + +-- Example successful test +insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) +with test_key as ( + select 'test' test_domain, + 'test_success' test_name from dual + ) +select 1 test_value, test_key.*, sq_result_id.nextval, sysdate from test_key; + +-- Example test with no bounds specified +insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) +with test_key as ( + select 'test' test_domain, + 'test_no_bound' test_name from dual + ) +select 1 test_value, test_key.*, sq_result_id.nextval, sysdate from test_key; + + +/* complete check */ +select 1 complete +from (select test_name from etl_test_values where 1=0 + union all select * from dual -- guarantee 1 row + ); diff --git a/Oracle/migrate_fact_upload.sql b/Oracle/migrate_fact_upload.sql new file mode 100644 index 0000000..9dc8855 --- /dev/null +++ b/Oracle/migrate_fact_upload.sql @@ -0,0 +1,22 @@ +/** migrate_fact_upload - append data from a workspace table. +*/ + +insert /*+ parallel(&¶llel_degree) append */ into &&I2B2STAR.observation_fact +select * from &&workspace_star.observation_fact_&&upload_id ; + +insert into &&I2B2STAR.upload_status +select * from &&workspace_star.upload_status where upload_id = &&upload_id +-- Handle the case where workspace_star and I2B2STAR are the same. +and upload_id not in (select upload_id from &&I2B2STAR.upload_status) +; + +update &&I2B2STAR.upload_status set load_status = 'OK' +where upload_id = &&upload_id; + +commit ; + + +select 1 complete +from "&&I2B2STAR".upload_status +where upload_id = &&upload_id +; diff --git a/README.md b/README.md index 2730593..f007f88 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,54 @@ -# i2p-transform -i2b2 to PCORnet Common Data Model Transformation - requires i2b2 PCORnet ontology +# HERON ETL to i2b2 + +We (*@@TODO aim to*) transform the data from a number of sources and +load it into an i2b2 data repository, following the +[i2b2 Data Repository (CRC) Cell Design Document][CRC]. + +[CRC]: https://www.i2b2.org/software/files/PDF/current/CRC_Design.pdf + +## Usage + +To build flowsheet transformation views, +once installation and configuration (below) is done, invoke the +`FlowsheetViews` task following [luigi][] conventions: + + luigi --module epic_flowsheets FlowsheetViews + +In more detail: + + PYTHONPATH=. LUIGI_CONFIG_PATH=heron-test.cfg luigi \ + --local-scheduler \ + --module epic_flowsheets FlowsheetViews + +*@@TODO: actual fact loading* + +Then use `etl_tasks.MigratePendingUploads` to install the data in the +production i2b2 `observation_fact` table. Use +`@@TODO.PatientDimension` and `@@TODO.VisitDimLoad` to load the +`patient_dimension` and `visit_dimension` tables from +`observation_fact`. + +[luigi]: https://github.com/spotify/luigi + +Troubleshooting is discussed in [CONTRIBUTING][]. + +## Installation, Dependencies + +See `requirements.txt`. + +### luigid (optional) + + *@@TODO elaborate on this once we establish production conventions.* + + docker run --name luigid -p8082:8082 -v/STATE:/luigi/state -v/CONFIG:/etc/luigi -t stockport/luigid + + +## Configuration + +See [client.cfg](client.cfg). + +## Design and Development + +For design info and project coding style, see [CONTRIBUTING][]. + +[CONTRIBUTING]: CONTRIBUTING.md diff --git a/client.cfg b/client.cfg new file mode 100644 index 0000000..1a925da --- /dev/null +++ b/client.cfg @@ -0,0 +1,58 @@ +# Usage: +# +# Copy client.cfg to your.cfg and set LUIGI_CONFIG_PATH=your.cfg. + +[core] +logging_conf_file = logging.cfg + +[ETLAccount] +# Account identifier in SQLAlchemy URL format. Be sure this includes +# all and only those details that identify the ETL +# account. Insignificant details such as password and tunnel port go +# in other parameters; including them here would result in +# inconsistent task identifiers. +# +# WRONG: +# oracle://username:password@localhost:5555/database_sid +# +# RIGHT: +account=oracle://grouse_etl_1@dbhost2/database_sid +passkey=GROUSE_ETL_1_ON_DBHOST2 +ssh_tunnel=localhost:4768 + + +[CLARITYExtract] +# When was it downloaded? +# e.g. suppose a jenkins download job was http://.../job/cms_syn_dl/8 +# and it finished Jul 30, 2015 8:54:25 AM. The we have: +download_date=1487378515445 + +# In order to get build dates from jenkins to luigi, i.e. +# from groovy to python, we use integers, since date interchange +# is a pain. +# +# Using the Jenkins API http://javadoc.jenkins.io/ +# A bit of groovy like this gets what we need: +# dlBuild?.timestamp.getTimeInMillis() + dlBuild?.duration + +# TODO: Split work into how many chunks by bene_id? +# bene_chunks = 64 + + +[NightHeronCreate] +# Where did we (or should we) create an i2b2 project? +# ref i2b2 sources: +# crc_create_datamart_oracle.sql +# crc_create_uploader_oracle.sql +star_schema = NIGHTHERONDATA + +# And what i2b2 project_id? +project_id = BlueHeron + +#[scheduler] +# ISSUE: task history? +#record_task_history = true + +[resources] +encounter_mapping=1 +patient_mapping=1 diff --git a/epic_flowsheets.py b/epic_flowsheets.py new file mode 100644 index 0000000..1466131 --- /dev/null +++ b/epic_flowsheets.py @@ -0,0 +1,192 @@ +'''epic_flowsheets -- ETL tasks for Epic Flowsheets to i2b2 +''' + +from typing import List + +import luigi +import sqlalchemy as sqla + +from etl_tasks import ( + DBAccessTask, I2B2ProjectCreate, I2B2Task, SourceTask, + SqlScriptTask, UploadTask, + DBTarget, SchemaTarget, UploadTarget +) +from script_lib import Script +from sql_syntax import Environment, Params +import param_val as pv + + +class CLARITYExtract(SourceTask, DBAccessTask): + download_date = pv.TimeStampParam(description='see client.cfg') + source_cd = pv.StrParam(default="Epic@kumed.com") + + # ISSUE: parameterize CLARITY schema name? + schema = 'CLARITY' + table_eg = 'patient' + + def _dbtarget(self) -> DBTarget: + return SchemaTarget(self._make_url(self.account), + schema_name=self.schema, + table_eg=self.table_eg, + echo=self.echo) + + +class NightHeronCreate(I2B2ProjectCreate): + pass + + +class NightHeronTask(I2B2Task): + '''Mix in identified star_schema parameter config. + ''' + @property + def project(self) -> I2B2ProjectCreate: + return NightHeronCreate() + + +class FromEpic(NightHeronTask): + '''Mix in source and substitution variables for Epic ETL scripts. + ''' + @property + def source(self) -> CLARITYExtract: + return CLARITYExtract() + + @property + def variables(self) -> Environment: + return self.vars_for_deps + + @property + def vars_for_deps(self) -> Environment: + # TODO: config = [] + # TODO: design = [] # TODO + return dict() + + +class FlowsheetViews(SqlScriptTask): + # TODO: subsume this in a load task + # TODO: patient survey using ntile() -- persistent worthwhile? + script = Script.epic_flowsheets_transform + + +class EpicDimensionsLoad(FromEpic, DBAccessTask): + '''Placeholder for heron_load/load_epic_dimensions.sql + ''' + @property + def transform_name(self) -> str: + return 'load_epic_dimensions' + + def complete(self) -> bool: + return self.output().exists() + + def output(self) -> luigi.Target: + return self._upload_target() + + def _upload_target(self) -> 'UploadTarget': + return UploadTarget(self._make_url(self.account), + self.project.upload_table, + self.transform_name, self.source, + echo=self.echo) + + def run(self) -> None: + raise NotImplementedError + + +class PatIdMapping(luigi.WrapperTask): + def requires(self) -> List[luigi.Task]: + # TODO: split pat_id mapping out of load_epic_dimensions.sql + return [EpicDimensionsLoad()] + + +class FSDMapping(luigi.WrapperTask): + def requires(self) -> List[luigi.Task]: + # TODO: split pat_id mapping out of load_epic_dimensions.sql + return [EpicDimensionsLoad()] + + +class EpicFactsLoadGroup(FromEpic, UploadTask): + epic_fact_view = pv.StrParam() + pat_id_lo = pv.StrParam() + pat_id_hi = pv.StrParam() + pat_group_num = pv.IntParam(significant=False) + pat_group_qty = pv.IntParam(significant=False) + pat_source_cd = pv.StrParam(default='Epic@kumed.com') + enc_source_cd = pv.StrParam() + + script = Script.epic_facts_load + + @property + def label(self) -> str: + return '{view} #{group}: {lo} to {hi}'.format( + view=self.epic_fact_view, group=self.pat_group_num, + lo=self.pat_id_lo, hi=self.pat_id_hi) + + def requires(self) -> List[luigi.Task]: + return UploadTask.requires(self) + [ + self.source, + FSDMapping(), + PatIdMapping(), + ] + + @property + def variables(self) -> Environment: + return dict(self.vars_for_deps, + epic_fact_view=self.epic_fact_view, + log_fact_exceptions='') + + def script_params(self) -> Params: + return dict(UploadTask.script_params(self), + pat_source_cd=self.pat_source_cd, + enc_source_cd=self.enc_source_cd, + pat_id_lo=self.pat_id_lo, + pat_id_hi=self.pat_id_hi) + + +class FlowsheetsLoad(FromEpic, DBAccessTask, luigi.WrapperTask): + pat_group_qty = pv.IntParam(default=5, significant=False) + enc_source_cd = pv.StrParam(default='Epic+pat_id_day@kumed.com') + + # issue: parameterize fact views? + fact_views = [ + 'numerictypeflows', + 'numerictypeflows', + 'datemeasureflows', + 'selectflows', + 'idstringtypeflows', + 'deidstringtypeflows', + ] + + pat_grp_q = ''' + select :group_qty grp_qty, group_num + , min(pat_id) pat_id_lo + , max(pat_id) pat_id_hi + from ( + select pat_id + , ntile(:group_qty) over (order by pat_id) as group_num + from ( + select /*+ parallel(20) */ distinct pat_id + from clarity.patient + where pat_id is not null -- help Oracle use the index + ) ea + ) w_ntile + group by group_num, :group_qty + order by group_num + ''' + + def requires(self) -> List[luigi.Task]: + groups = self.partition_patients() + return [ + EpicFactsLoadGroup(epic_fact_view=view, + enc_source_cd=self.enc_source_cd, + pat_id_lo=lo, + pat_id_hi=hi, + pat_group_qty=qty, + pat_group_num=num) + for view in self.fact_views + for (qty, num, lo, hi) in groups] + + def partition_patients(self) -> List[sqla.engine.RowProxy]: + # ISSUE: persist partition? + with self.connection('partition patients') as q: + groups = q.execute(self.pat_grp_q.format(i2b2_star=self.project.star_schema), + params=dict(group_qty=self.pat_group_qty)).fetchall() + q.log.info('groups: %s', groups) + return groups diff --git a/etl_tasks.py b/etl_tasks.py new file mode 100644 index 0000000..9e01043 --- /dev/null +++ b/etl_tasks.py @@ -0,0 +1,897 @@ +'''etl_tasks -- Source-agnostic Luigi ETL Task support + +Note: This is source-agnostic but not target-agnositc; it has i2b2 + knowledge. + +''' + +from typing import Any, Callable, Dict, Iterator, List, Optional as Opt, Tuple, cast +from contextlib import contextmanager +from datetime import datetime +import csv +import logging + +from luigi.contrib.sqla import SQLAlchemyTarget +from sqlalchemy import text as sql_text, func, Table # type: ignore +from sqlalchemy.engine import Connection, Engine +from sqlalchemy.engine.result import ResultProxy +from sqlalchemy.engine.url import make_url +from sqlalchemy.exc import DatabaseError +from sqlalchemy.sql.expression import Select +import sqlalchemy as sqla +import luigi +from cx_Oracle import Error as OraError, _Error as Ora_Error + +from eventlog import EventLogger, LogState, JSONObject # ISSUE: use https://eliot.readthedocs.io/en/1.1.0/ instead? +from param_val import StrParam, IntParam, BoolParam +from script_lib import Script +from sql_syntax import Environment, Params, SQL +from sql_syntax import params_used, insert_append_table + +log = logging.getLogger(__name__) + + +class DBTarget(SQLAlchemyTarget): + '''Take advantage of engine caching logic from SQLAlchemyTarget, + but don't bother with target_table, update_id, etc. + + >>> t = DBTarget(connection_string='sqlite:///') + >>> t.engine.scalar('select 1 + 1') + 2 + ''' + def __init__(self, connection_string: str, + target_table: Opt[str]=None, update_id: Opt[str]=None, + echo: bool=False) -> None: + SQLAlchemyTarget.__init__( + self, connection_string, + target_table=target_table, + update_id=update_id, + echo=echo) + + def exists(self) -> bool: + raise NotImplementedError + + def touch(self) -> None: + raise NotImplementedError + + +class ETLAccount(luigi.Config): + '''Access to connect and run ETL. + + This account needs read access to source material, write access to + the destination i2b2 star schema, and working space for + intermediary tables, views, functions, and such. + ''' + account = StrParam(description='see client.cfg', + default='') + passkey = StrParam(description='see client.cfg', + default='', + significant=False) + ssh_tunnel = StrParam(description='see client.cfg', + default='', + significant=False) + echo = BoolParam(description='SQLAlchemy echo logging', + significant=False) + + +class LoggedConnection(object): + '''Wrap (parts of) the sqlalchemy Connection API with logging. + + If you have an EventLogger, `log`, you can wrap a connection + in a step that explains what the connection is for:: + + with log.step('crunching data') as step: + lc = LoggedConnection(conn, log, step) + + .. note:: A LoggedConnection wraps only the `execute` and `scalar` + methods from the sqlalchemy API. + + ''' + def __init__(self, conn: Connection, log: EventLogger, + step: LogState) -> None: + self._conn = conn + self.log = log + self.step = step + + def __repr__(self) -> str: + return '%s(%s, %s)' % (self.__class__.__name__, self._conn, self.log) + + def _log_args(self, event: str, operation: object, + params: Params) -> Tuple[str, JSONObject, JSONObject]: + msg = '%(event)s %(sql3)s' + ('\n%(params)s' if params else '') + argobj = dict(event=event, sql3=_peek(operation, lines=3), params=params) + extra = dict(statement=str(operation)) + return msg, argobj, extra + + def execute(self, operation: object, params: Opt[Params] = None) -> ResultProxy: + msg, argobj, extra = self._log_args('execute', operation, params or {}) + with self.log.step(msg, argobj, extra): + return self._conn.execute(operation, params or {}) + + def scalar(self, operation: object, params: Opt[Params] = None) -> Any: + msg, argobj, extra = self._log_args('scalar', str(operation), params or {}) + with self.log.step(msg, argobj, extra) as step: + result = self._conn.scalar(operation, params or {}) + step.extra.update(dict(result=result)) + return result + + +def _peek(thing: object, + lines: int=1, + max_len: int=120) -> str: + return '\n'.join(str(thing).split('\n')[:lines])[:180] + + +class DBAccessTask(luigi.Task): + """Manage DB account credentials and logging. + + Typical usage:: + + with self.connection(event='munching data') as conn: + howmany = conn.scalar('select count(*) from cookies') + yummy = conn.execute('select * from cookies') + """ + account = StrParam(default=ETLAccount().account) + ssh_tunnel = StrParam(default=ETLAccount().ssh_tunnel, + significant=False) + passkey = StrParam(default=ETLAccount().passkey, + significant=False) + echo = BoolParam(default=ETLAccount().echo, + significant=False) + max_idle = IntParam(description='Set to less than Oracle profile max idle time.', + default=60 * 20, + significant=False) + _log = logging.getLogger(__name__) # ISSUE: ambient. + + def output(self) -> luigi.Target: + return self._dbtarget() + + def _dbtarget(self) -> DBTarget: + return DBTarget(self._make_url(self.account), + target_table=None, update_id=self.task_id, + echo=self.echo) + + def _make_url(self, account: str) -> str: + url = make_url(account) + if 'oracle' in account.lower(): + url.query['pool_recycle'] = self.max_idle + # `twophase` interferes with direct path load somehow. + url.query['allow_twophase'] = False + if self.passkey: + from os import environ # ISSUE: ambient + url.password = environ[self.passkey] + if self.ssh_tunnel and url.host: + host, port = self.ssh_tunnel.split(':', 1) + url.host = host + url.port = port + return str(url) + + def log_info(self) -> Dict[str, Any]: + '''Get info to log: luigi params, task_family, task_hash. + ''' + return dict(self.to_str_params(only_significant=True), + task_family=self.task_family, + task_hash=self.task_id[-luigi.task.TASK_ID_TRUNCATE_HASH:]) + + @contextmanager + def connection(self, event: str='connect') -> Iterator[LoggedConnection]: + conn = ConnectionProblem.tryConnect(self._dbtarget().engine) + log = EventLogger(self._log, self.log_info()) + with log.step('%(event)s: <%(account)s>', + dict(event=event, account=self.account)) as step: + yield LoggedConnection(conn, log, step) + + def _fix_password(self, environ: Dict[str, str], getpass: Callable[[str], str]) -> None: + '''for interactive use; e.g. in notebooks + ''' + if self.passkey not in environ: + environ[self.passkey] = getpass(self.passkey) + + +class SqlScriptTask(DBAccessTask): + '''Task to run a stylized SQL script. + + As seen in `script_lib`, a script may require (in the luigi sense) + other scripts and it is complete iff its last query says so. + + Running a script may be parameterized with bind params and/or + Oracle sqlplus style defined `&&variables`: + + >>> variables = dict(I2B2STAR='I2B2DEMODATA') + >>> txform = SqlScriptTask( + ... account='sqlite:///', passkey=None, + ... script=Script.migrate_fact_upload, + ... param_vars=variables) + + ISSUE: doctest dependencies? + >>> [task.script for task in txform.requires()] + ... #doctest: +ELLIPSIS + [] + + >>> txform.complete() + False + + ''' + script = cast(Script, luigi.EnumParameter(enum=Script)) + param_vars = cast(Environment, luigi.DictParameter(default={})) + _log = logging.getLogger('sql_scripts') # ISSUE: ambient. magic-string + + @property + def variables(self) -> Environment: + '''Defined variables for this task (or task family). + ''' + return self.param_vars + + @property + def vars_for_deps(self) -> Environment: + '''Defined variables to supply to dependencies. + ''' + return self.variables + + def requires(self) -> List[luigi.Task]: + '''Wrap each of `self.script.deps()` in a SqlScriptTask. + ''' + return [SqlScriptTask(script=s, + param_vars=self.vars_for_deps, + account=self.account, + passkey=self.passkey, + echo=self.echo) + for s in self.script.deps()] + + def log_info(self) -> Dict[str, Any]: + '''Include script, filename in self.log_info(). + ''' + return dict(DBAccessTask.log_info(self), + script=self.script.name, + filename=self.script.fname) + + def complete(self) -> bool: + '''Each script's last query tells whether it is complete. + + It should be a scalar query that returns non-zero for done + and either zero or an error for not done. + ''' + last_query = self.last_query() + params = params_used(self.complete_params(), last_query) + with self.connection(event=self.task_family + ' complete query: ' + self.script.name) as conn: + try: + result = conn.scalar(sql_text(last_query), params) + return bool(result) + except DatabaseError as exc: + conn.log.warning('%(event)s: %(exc)s', + dict(event='complete query error', exc=exc)) + return False + + def last_query(self) -> SQL: + """ + Note: In order to support run-only variables as in UploadTask, + we skip statements with unbound &&variables. + """ + return self.script.statements( + skip_unbound=True, + variables=self.variables)[-1] + + def complete_params(self) -> Dict[str, Any]: + '''Make `task_id` available to complete query as a bind param. + ''' + return dict(task_id=self.task_id) + + def run(self) -> None: + '''Run each statement in the script without any bind parameters. + ''' + self.run_bound() + + def run_bound(self, + script_params: Opt[Params]=None) -> None: + '''Run with a (default emtpy) set of parameters bound. + ''' + with self.connection(event='run script') as conn: + self.run_event(conn, script_params=script_params) + + def run_event(self, + conn: LoggedConnection, + run_vars: Opt[Environment]=None, + script_params: Opt[Params]=None) -> int: + '''Run script inside a LoggedConnection event. + + @param run_vars: variables to define for this run + @param script_params: parameters to bind for this run + @return: count of rows bulk-inserted + always 0 for this class, but see UploadTask + + To see how a script can ignore errors, see :mod:`script_lib`. + ''' + bulk_rows = 0 + ignore_error = False + run_params = dict(script_params or {}, task_id=self.task_id) + fname = self.script.fname + variables = dict(run_vars or {}, **self.variables) + each_statement = self.script.each_statement(variables=variables) + + for line, _comment, statement in each_statement: + try: + if self.is_bulk(statement): + bulk_rows = self.bulk_insert( + conn, fname, line, statement, run_params, + bulk_rows) + else: + ignore_error = self.execute_statement( + conn, fname, line, statement, run_params, + ignore_error) + except DatabaseError as exc: + db = self._dbtarget().engine + err = SqlScriptError(exc, self.script, line, + statement, str(db)) + if ignore_error: + conn.log.warning('%(event)s: %(error)s', + dict(event='ignore', error=err)) + else: + raise err from None + if bulk_rows > 0: + conn.step.msg_parts.append(' %(rowtotal)s total rows') + conn.step.argobj.update(dict(rowtotal=bulk_rows)) + + return bulk_rows + + def execute_statement(self, conn: LoggedConnection, fname: str, line: int, + statement: SQL, run_params: Params, + ignore_error: bool) -> bool: + '''Log and execute one statement. + ''' + sqlerror = Script.sqlerror(statement) + if sqlerror is not None: + return sqlerror + params = params_used(run_params, statement) + self.set_status_message( + '%s:%s:\n%s\n%s' % (fname, line, statement, params)) + conn.execute(statement, params) + return ignore_error + + def is_bulk(self, statement: SQL) -> bool: + '''always False for this class, but see UploadTask + ''' + return False + + def bulk_insert(self, conn: LoggedConnection, fname: str, line: int, + statement: SQL, run_params: Params, + bulk_rows: int) -> int: + raise NotImplementedError( + 'overriding is_bulk() requires overriding bulk_insert()') + + +def log_plan(lc: LoggedConnection, event: str, params: Dict[str, Any], + query: Opt[Select]=None, sql: Opt[str]=None) -> None: + if query is not None: + sql = str(query.compile(bind=lc._conn)) + if sql is None: + return + plan = explain_plan(lc, sql) + param_msg = ', '.join('%%(%s)s' % k for k in params.keys()) + lc.log.info('%(event)s [' + param_msg + ']\n' + 'query: %(query_peek)s plan:\n' + '%(plan)s', + dict(params, event=event, query_peek=_peek(sql), + plan='\n'.join(plan))) + + +def explain_plan(work: LoggedConnection, statement: SQL) -> List[str]: + work.execute('explain plan for ' + statement) + # ref 19 Using EXPLAIN PLAN + # Oracle 10g Database Performance Tuning Guide + # https://docs.oracle.com/cd/B19306_01/server.102/b14211/ex_plan.htm + plan = work.execute( + 'SELECT PLAN_TABLE_OUTPUT line FROM TABLE(DBMS_XPLAN.DISPLAY())') + return [row.line for row in plan] # type: ignore # sqla + + +def maybe_ora_err(exc: Exception) -> Opt[Ora_Error]: + if isinstance(exc, DatabaseError): + if isinstance(exc.orig, OraError): + return cast(Ora_Error, exc.orig.args[0]) + return None + + +class SqlScriptError(IOError): + '''Include script file, line number in diagnostics + ''' + def __init__(self, exc: Exception, script: Script, line: int, statement: SQL, + conn_label: str) -> None: + fname = script.name + message = '%s <%s>\n%s:%s:\n' + args = [exc, conn_label, fname, line] + ora_ex = maybe_ora_err(exc) + if ora_ex: + offset = ora_ex.offset + message += '%s%s' + args[0] = ora_ex.message + args += [_pick_lines(statement[:offset], -3, None), + _pick_lines(statement[offset:], None, 3)] + else: + message += '%s' + args += [statement] + + self.message = message + self.args = tuple(args) + + def __str__(self) -> str: + return self.message % self.args + + +def _pick_lines(s: str, lo: Opt[int], hi: Opt[int]) -> str: + return '\n'.join(s.split('\n')[lo:hi]) + + +class TimeStampParameter(luigi.Parameter): + '''A datetime interchanged as milliseconds since the epoch. + ''' + + def parse(self, s: str) -> datetime: + ms = int(s) + return datetime.fromtimestamp(ms / 1000.0) + + def serialize(self, dt: datetime) -> str: + epoch = datetime.utcfromtimestamp(0) + ms = (dt - epoch).total_seconds() * 1000 + return str(int(ms)) + + +class SourceTask(luigi.Task): + @property + def source_cd(self) -> str: + raise NotImplementedError + + @property + def download_date(self) -> datetime: + raise NotImplementedError + + +class I2B2Task(object): + @property + def project(self) -> 'I2B2ProjectCreate': + return I2B2ProjectCreate() + + +class UploadTask(I2B2Task, SqlScriptTask): + '''Run a script with an associated `upload_status` record. + ''' + @property + def source(self) -> SourceTask: + raise NotImplementedError('subclass must implement') + + @property + def transform_name(self) -> str: + return self.task_id + + def complete_params(self) -> Dict[str, Any]: + return dict(task_id=self.task_id, + download_date=self.source.download_date) + + def output(self) -> luigi.Target: + return self._upload_target() + + def _upload_target(self) -> 'UploadTarget': + return UploadTarget(self._make_url(self.account), + self.project.upload_table, + self.transform_name, self.source, + echo=self.echo) + + def requires(self) -> List[luigi.Task]: + return [self.project, self.source] + SqlScriptTask.requires(self) + + def complete(self) -> bool: + # Belt and suspenders + return (self.output().exists() and + SqlScriptTask.complete(self)) + + @property + def label(self) -> str: + return self.script.title + + def run(self) -> None: + upload = self._upload_target() + with upload.job(self, + label=self.label, + user_id=make_url(self.account).username) as conn_id_r: + conn, upload_id, result = conn_id_r + bulk_rows = SqlScriptTask.run_event( + self, conn, + run_vars=dict(upload_id=str(upload_id)), + script_params=dict(self.script_params(), upload_id=upload_id)) + result[upload.table.c.loaded_record.name] = bulk_rows + + def script_params(self) -> Params: + return dict(download_date=self.source.download_date, + project_id=self.project.project_id) + + def is_bulk(self, statement: SQL) -> bool: + return insert_append_table(statement) is not None + + def bulk_insert(self, conn: LoggedConnection, fname: str, line: int, + statement: SQL, run_params: Params, + bulk_rows: int) -> int: + with conn.log.step( + '%(filename)s:%(lineno)s: %(event)s', + dict(event='bulk_insert', + filename=fname, lineno=line)) as step: + plan = '\n'.join(explain_plan(conn, statement)) + conn.log.info('%(filename)s:%(lineno)s: %(event)s:\n%(plan)s', + dict(filename=fname, lineno=line, event='plan', + plan=plan)) + + params = params_used(run_params, statement) + self.set_status_message( + '%s:%s:\n%s\n%s' % (fname, line, statement, params)) + last_result = conn.execute(statement, params) + step.msg_parts.append(' %(rowcount)d rows') + step.argobj.update(dict(rowcount=last_result.rowcount)) + bulk_rows += last_result.rowcount + + return bulk_rows + + +class UploadTarget(DBTarget): + def __init__(self, connection_string: str, + table: sqla.Table, transform_name: str, source: SourceTask, + echo: bool=False) -> None: + DBTarget.__init__(self, connection_string, + echo=echo) + self.table = table + self.source = source + self.transform_name = transform_name + self.upload_id = None # type: Opt[int] + + def __repr__(self) -> str: + return '%s(transform_name=%s)' % ( + self.__class__.__name__, self.transform_name) + + def exists(self) -> bool: + conn = ConnectionProblem.tryConnect(self.engine) + with conn.begin(): + up_t = self.table + upload_id = conn.scalar( + sqla.select([sqla.func.max(up_t.c.upload_id)]) + .select_from(up_t) + .where(sqla.and_(up_t.c.transform_name == self.transform_name, + up_t.c.load_status == 'OK'))) + return upload_id is not None + + @contextmanager + def job(self, task: DBAccessTask, + label: Opt[str] = None, user_id: Opt[str] = None, + upload_id: Opt[int] = None) -> Iterator[ + Tuple[LoggedConnection, int, Params]]: + event = 'upload job' + with task.connection(event=event) as conn: + up_t = self.table + if upload_id is None: + if user_id is None: + raise TypeError('must supply user_id for new record') + if label is None: + raise TypeError('must supply label for new record') + upload_id = self.insert(conn, label, user_id) + else: + [label, user_id] = conn.execute( + sqla.select([up_t.c.upload_label, up_t.c.user_id]) + .where(up_t.c.upload_id == upload_id)).fetchone() + + msg = ' %(upload_id)s for %(label)s' + info = dict(label=label, upload_id=upload_id) + conn.step.msg_parts.append(msg) + conn.step.argobj.update(info) + conn.log.info(msg, info) # Go ahead and log the upload_id early. + + result = {} # type: Params + yield conn, upload_id, result + conn.execute(up_t.update() + .where(up_t.c.upload_id == upload_id) + .values(load_status='OK', end_date=func.now(), + **result)) + + def insert(self, conn: LoggedConnection, label: str, user_id: str) -> int: + ''' + :param label: a label for related facts for audit purposes + :param user_id: an indication of who uploaded the related facts + ''' + up_t = self.table + next_q = sql_text( + '''select {i2b2}.sq_uploadstatus_uploadid.nextval + from dual'''.format(i2b2=self.table.schema)) + upload_id = conn.scalar(next_q) # type: int + + conn.execute(up_t.insert() + .values(upload_id=upload_id, + upload_label=label, + user_id=user_id, + source_cd=self.source.source_cd, + load_date=sqla.func.now(), + transform_name=self.transform_name)) + self.upload_id = upload_id + return upload_id + + +class I2B2ProjectCreate(DBAccessTask): + star_schema = StrParam(description='see client.cfg') + project_id = StrParam(description='see client.cfg') + _meta = None # type: Opt[sqla.MetaData] + _upload_table = None # type: Opt[sqla.Table] + + Column, ty = sqla.Column, sqla.types + upload_status_columns = [ + Column('upload_id', ty.Numeric(38, 0, asdecimal=False), primary_key=True), + Column('upload_label', ty.String(500), nullable=False), + Column('user_id', ty.String(100), nullable=False), + Column('source_cd', ty.String(50), nullable=False), + Column('no_of_record', ty.Numeric(asdecimal=False)), + Column('loaded_record', ty.Numeric(asdecimal=False)), + Column('deleted_record', ty.Numeric(asdecimal=False)), + Column('load_date', ty.DateTime, nullable=False), + Column('end_date', ty.DateTime), + Column('load_status', ty.String(100)), + Column('message', ty.Text), + Column('input_file_name', ty.Text), + Column('log_file_name', ty.Text), + Column('transform_name', ty.String(500)), + ] + + def output(self) -> 'SchemaTarget': + return SchemaTarget(self._make_url(self.account), + schema_name=self.star_schema, + table_eg='patient_dimension', + echo=self.echo) + + def run(self) -> None: + raise NotImplementedError('see heron_create.create_deid_datamart etc.') + + @property + def metadata(self) -> sqla.MetaData: + if self._meta: + return self._meta + self._meta = meta = sqla.MetaData(schema=self.star_schema) + return meta + + def table_details(self, lc: LoggedConnection, tables: List[str]) -> sqla.MetaData: + i2b2_meta = sqla.MetaData(schema=self.star_schema) + i2b2_meta.reflect(only=tables, schema=self.star_schema, + bind=self._dbtarget().engine) + return i2b2_meta + + @property + def upload_table(self) -> sqla.Table: + if self._upload_table is not None: + return self._upload_table + t = sqla.Table( + 'upload_status', self.metadata, + *self.upload_status_columns, + schema=self.star_schema) + self._upload_table = t + return t + + +class SchemaTarget(DBTarget): + def __init__(self, connection_string: str, schema_name: str, table_eg: str, + echo: bool=False) -> None: + DBTarget.__init__(self, connection_string, echo=echo) + self.schema_name = schema_name + self.table_eg = table_eg + + def exists(self) -> bool: + table = Table(self.table_eg, sqla.MetaData(), schema=self.schema_name) + return table.exists(bind=self.engine) # type: ignore + + +class ConnectionProblem(DatabaseError): + '''Provide hints about ssh tunnels. + ''' + # connection closed, no listener + tunnel_hint_codes = [12537, 12541] + + @classmethod + def tryConnect(cls, engine: Engine) -> Connection: + try: + return engine.connect() + except DatabaseError as exc: + raise ConnectionProblem.refine(exc, str(engine)) from None + + @classmethod + def refine(cls, exc: Exception, conn_label: str) -> Exception: + '''Recognize known connection problems. + + :returns: customized exception for known + problem else exc + ''' + ora_ex = maybe_ora_err(exc) + + if ora_ex: + return cls(exc, ora_ex, conn_label) + return exc + + def __init__(self, exc: DatabaseError, ora_ex: Ora_Error, conn_label: str) -> None: + DatabaseError.__init__( + self, + exc.statement, exc.params, + exc.connection_invalidated) + message = '%s <%s>' + args = [ora_ex, conn_label] + + if exc.statement and ora_ex.offset: + stmt_rest = exc.statement[ + ora_ex.offset:ora_ex.offset + 120] + message += '\nat: %s' + args += [stmt_rest] + local_conn_prob = ( + ora_ex.code in self.tunnel_hint_codes and + 'localhost' in conn_label) + if local_conn_prob: + message += '\nhint: ssh tunnel down?' + message += '\nin: %s' + args += [ora_ex.context] + self.message = message + self.args = tuple(args) + + def __str__(self) -> str: + return self.message % self.args + + +class ReportTask(DBAccessTask): + @property + def script(self) -> Script: + raise NotImplementedError('subclass must implement') + + @property + def report_name(self) -> str: + raise NotImplementedError('subclass must implement') + + def complete(self) -> bool: + '''Double-check requirements as well as output. + ''' + deps = luigi.task.flatten(self.requires()) # type: List[luigi.Task] + return (self.output().exists() and + all(t.complete() for t in deps)) + + def _csvout(self) -> 'CSVTarget': + return CSVTarget(path=self.report_name + '.csv') + + def output(self) -> luigi.Target: + return self._csvout() + + def run(self) -> None: + with self.connection('report') as conn: + query = sql_text( + 'select * from {object}'.format(object=self.report_name)) + result = conn.execute(query) + cols = result.keys() + rows = result.fetchall() + self._csvout().export(cols, rows) + + +class CSVTarget(luigi.local_target.LocalTarget): + def export(self, cols: List[str], data: List) -> None: + with self.open('wb') as stream: + dest = csv.writer(stream) + dest.writerow(cols) + dest.writerows(data) + + @contextmanager + def dictreader(self, + lowercase_fieldnames: bool=False, + delimiter: str=',') -> Iterator[csv.DictReader]: + '''DictReader contextmanager + + @param lowercase_fieldnames: sqlalchemy uses lower-case bind + parameter names, but SCILHS CSV file headers use the + actual uppercase column names. So we got: + + CompileError: The 'oracle' dialect with current + database version settings does not support empty + inserts. + + + ''' + with self.open('rb') as stream: + dr = csv.DictReader(stream, delimiter=delimiter) + if lowercase_fieldnames: + # This is a bit of a kludge, but it works... + dr.fieldnames = [n.lower() for n in dr.fieldnames] + yield dr + + +class AdHoc(DBAccessTask): + sql = StrParam() + name = StrParam() + + def _csvout(self) -> CSVTarget: + return CSVTarget(path=self.name + '.csv') + + def output(self) -> luigi.Target: + return self._csvout() + + def run(self) -> None: + with self.connection() as work: + result = work.execute(self.sql) + cols = result.keys() + rows = result.fetchall() + self._csvout().export(cols, rows) + + +class KillSessions(DBAccessTask): + reason = StrParam(default='*no reason given*') + sql = ''' + begin + sys.kill_own_sessions(:reason); + end; + ''' + + def complete(self) -> bool: + return False + + def run(self) -> None: + with self.connection('kill own sessions') as work: + work.execute(self.sql, params=dict(reason=self.reason)) + + +class AlterStarNoLogging(DBAccessTask): + sql = ''' + alter table TABLE nologging + ''' + tables = ['patient_mapping', + 'encounter_mapping', + 'patient_dimension', + 'visit_dimension', + 'observation_fact'] + + def complete(self) -> bool: + return False + + def run(self) -> None: + with self.connection() as work: + for table in self.tables: + work.execute(self.sql.replace('TABLE', table)) + + +class MigrateUpload(SqlScriptTask, I2B2Task): + upload_id = IntParam() + workspace_star = StrParam() + parallel_degree = IntParam(default=24, + significant=False) + + script = Script.migrate_fact_upload + + @property + def variables(self) -> Environment: + return dict(I2B2STAR=self.project.star_schema, + workspace_star=self.workspace_star, + parallel_degree=str(self.parallel_degree), + upload_id=str(self.upload_id)) + + +class MigratePendingUploads(DBAccessTask, I2B2Task, luigi.WrapperTask): + workspace_star = StrParam() + + find_pending = """ + select upload_id from %(WORKSPACE)s.upload_status + where load_status in ('OK', 'OK_work') and upload_id not in ( + select upload_id from %(I2B2STAR)s.upload_status + where load_status='OK' ) + """ + + def requires(self) -> List[luigi.Task]: + find_pending = self.find_pending % dict( + WORKSPACE=self.workspace_star, + I2B2STAR=self.project.star_schema) + + deps = [] # type: List[luigi.Task] + with self.connection('pending uploads') as lc: + pending = [row.upload_id for row in + lc.execute(find_pending).fetchall()] + + workmeta = sqla.MetaData() + for upload_id in pending: + table = Table('observation_fact_%d' % upload_id, workmeta, + schema=self.workspace_star) + if table.exists(bind=lc._conn): + deps.append( + MigrateUpload(upload_id=upload_id, + workspace_star=self.workspace_star)) + else: + log.warn('no such table to migrate: %s', table) + return deps diff --git a/eventlog.py b/eventlog.py new file mode 100644 index 0000000..e8b2e3d --- /dev/null +++ b/eventlog.py @@ -0,0 +1,152 @@ +'''eventlog -- structured logging support + +EventLogger for nested events +============================= + +Suppose we want to log some events which naturally nest. + +First, add a handler to a logger: + +>>> import logging, sys +>>> log1 = logging.getLogger('log1') +>>> detail = logging.StreamHandler(sys.stdout) +>>> log1.addHandler(detail) +>>> log1.setLevel(logging.INFO) +>>> detail.setFormatter(logging.Formatter( +... fmt='%(levelname)s %(elapsed)s %(do)s %(message)s')) + +>>> io = MockIO() +>>> event0 = EventLogger(log1, dict(customer='Jones', invoice=123), io.clock) + + +>>> with event0.step('Build %(product)s', dict(product='house')): +... with event0.step('lay foundation %(depth)d ft deep', +... dict(depth=20)) as info: +... info.msg_parts.append(' at %(temp)d degrees') +... info.argobj['temp'] = 65 +... with event0.step('frame %(stories)d story house', +... dict(stories=2)) as info: +... pass +... start, elapsed, _ms = event0.elapsed() +... eta = event0.eta(pct=25) +... # doctest: +ELLIPSIS +INFO ('... 12:30:01', None, None) begin 0:00:00 [1] Build house... +INFO ('... 12:30:03', None, None) begin 0:00:02 [1, 2] lay foundation 20 ft deep... +INFO ('... 12:30:03', '0:00:03', 3000000) end 0:00:03 [1, 2] lay foundation 20 ft deep at 65 degrees. +INFO ('... 12:30:10', None, None) begin 0:00:09 [1, 3] frame 2 story house... +INFO ('... 12:30:10', '0:00:05', 5000000) end 0:00:05 [1, 3] frame 2 story house. +INFO ('... 12:30:01', '0:00:35', 35000000) end 0:00:35 [1] Build house. + +>>> eta +datetime.datetime(2000, 1, 1, 12, 31, 49) + +''' + +from contextlib import contextmanager +from datetime import datetime +from typing import ( + Any, Callable, Dict, Iterator, List, MutableMapping, + NamedTuple, Optional as Opt, TextIO, Tuple +) +import logging + + +KWArgs = MutableMapping[str, Any] +JSONObject = Dict[str, Any] + +LogState = NamedTuple('LogState', [ + ('msg_parts', List[str]), + ('argobj', JSONObject), + ('extra', JSONObject)]) + + +class EventLogger(logging.LoggerAdapter): + def __init__(self, logger: logging.Logger, event: JSONObject, + clock: Opt[Callable[[], datetime]]=None) -> None: + logging.LoggerAdapter.__init__(self, logger, extra={}) + self.name = logger.name + if clock is None: + clock = datetime.now # ISSUE: ambient + self.event = event + self._clock = clock + self._seq = 0 + self._step = [] # type: List[Tuple[int, datetime]] + List # let flake8 know we're using it + + def __repr__(self) -> str: + return '%s(%s, %s)' % (self.__class__.__name__, self.name, self.event) + + def process(self, msg: str, kwargs: KWArgs) -> Tuple[str, KWArgs]: + extra = dict(kwargs.get('extra', {}), + context=self.event) + return msg, dict(kwargs, extra=extra) + + def elapsed(self, then: Opt[datetime]=None) -> Tuple[str, str, int]: + start = then or self._step[-1][1] + elapsed = self._clock() - start + ms = int(elapsed.total_seconds() * 1000000) + return (str(start), str(elapsed), ms) + + def eta(self, pct: float) -> datetime: + t0 = self._step[0][1] + elapsed = self._clock() - t0 + return t0 + elapsed * (1 / (pct / 100)) + + @contextmanager + def step(self, msg: str, argobj: Dict[str, object], + extra: Opt[Dict[str, object]]=None) -> Iterator[LogState]: + checkpoint = self._clock() + self._seq += 1 + self._step.append((self._seq, checkpoint)) + extra = extra or {} + fmt_step = '%(t_step)s %(step)s ' + step_ixs = [ix for (ix, _t) in self._step] + t_step = str(checkpoint - self._step[0][1]) + self.info(fmt_step + msg + '...', + dict(argobj, step=step_ixs, t_step=t_step), + extra=dict(extra, do='begin', + elapsed=(str(checkpoint), None, None))) + msgparts = [msg] + outcome = logging.INFO + try: + yield LogState(msgparts, argobj, extra) + except: + outcome = logging.ERROR + raise + finally: + elapsed = self.elapsed(then=checkpoint) + self.log(outcome, ''.join([fmt_step] + msgparts) + '.', + dict(argobj, step=step_ixs, t_step=elapsed[1]), + extra=dict(extra, do='end', + elapsed=elapsed)) + self._step.pop() + + +class TextFilter(logging.Filter): + def __init__(self, skips: List[str]) -> None: + self.skips = skips + + def filter(self, record: logging.LogRecord) -> bool: + for skip in self.skips: + if record.getMessage().startswith(skip): + return False + return True + + +class TextHandler(logging.StreamHandler): + def __init__(self, stream: TextIO, skips: List[str]=[]) -> None: + logging.StreamHandler.__init__(self, stream) + self.addFilter(TextFilter(skips)) + + +class MockIO(object): + def __init__(self, + now: datetime=datetime(2000, 1, 1, 12, 30, 0)) -> None: + self._now = now + self._delta = 1 + + def clock(self) -> datetime: + from datetime import timedelta + self._now += timedelta(seconds=self._delta) + self._delta += 1 + return self._now diff --git a/logging.cfg b/logging.cfg new file mode 100644 index 0000000..2e6ec02 --- /dev/null +++ b/logging.cfg @@ -0,0 +1,81 @@ +# see section 15.8.3. Configuration file format in +# http://docs.python.org/2/library/logging.config.html +# +# also: +# https://github.com/pysysops/docker-luigi-taskrunner/blob/master/etc/luigi/logging.cfg +# https://pypi.python.org/pypi/python-json-logger +# https://stedolan.github.io/jq/manual/v1.5/ +# +# Logging to JSON lets us do fun stuff such as: +# +# $ < log/grouse-detail.json jq --compact-output -C \ +# 'select(.args|objects|.event == "inserted chunk") | +# [.asctime, .process, .args.filename, .args.lineno, .args.into, +# {elapsed: (.elapsed[1] | split("."))[0], +# rowcount: .args.rowcount, +# krowpersec: (.args.rowcount / .elapsed[2] * 1000 * 60 + 0.5 | floor)}]' +# ["2017-03-16 21:20:24",10770,"cms_patient_dimension.sql",9,"\"DCONNOLLY\".patient_dimension", +# {"elapsed":"0:01:39","rowcount":419835,"krowpersec":252}] +# ["2017-03-16 21:22:12",10770,"cms_patient_dimension.sql",9,"\"DCONNOLLY\".patient_dimension", +# {"elapsed":"0:01:47","rowcount":419835,"krowpersec":234}] + + +[loggers] +keys=root,luigi_debug + +[logger_root] +level=INFO +handlers=console,detail + +[logger_luigi_debug] +level=DEBUG +handlers=luigi_debug +qualname=luigi-interface + +[handlers] +keys=console,detail,luigi_debug + +[handler_console] +level=INFO +# class=StreamHandler +class=eventlog.TextHandler +args=(sys.stderr, ['execute', 'execute commit', 'complete query', 'find chunks']) +# args=(3, 0.1) +formatter=timed + +[handler_detail] +level=INFO +class=FileHandler +# use detail_log_dir rather than dir to facilitate stream editing +detail_log_dir=log +detail_log_file=%(detail_log_dir)s/grouse-detail.json +# append to log file +#args=('%(detail_log_file)s', 'a', None, True) +# overwrite log file +args=('%(detail_log_file)s', 'w') +formatter=json + +[handler_luigi_debug] +level=DEBUG +class=FileHandler +luigi_debug_log_dir=log +luigi_debug_log_file=%(luigi_debug_log_dir)s/grouse-luigi-debug.json +# append to log file +# args=('%(luigi_debug_log_file)s', 'a') +# overwrite log file +args=('%(luigi_debug_log_file)s', 'w') +formatter=json + +[formatters] +keys=timed, json + +[formatter_timed] +class=logging.Formatter +# %(name)s? +format=%(asctime)s %(process)s %(levelname)s: %(message)s +datefmt=%02H:%02M:%02S + +[formatter_json] +class = pythonjsonlogger.jsonlogger.JsonFormatter +format=%(asctime)s %(process)s %(name) %(levelname): %(message)s %(args)s +datefmt=%Y-%m-%02d %02H:%02M:%02S diff --git a/param_val.py b/param_val.py new file mode 100644 index 0000000..1b4c4aa --- /dev/null +++ b/param_val.py @@ -0,0 +1,37 @@ +# TODO: module doc: luigi's abuse of class attributes vs. mypy static typing +from datetime import datetime +from typing import Any, Callable, TypeVar, cast + +import luigi + + +_T = TypeVar('_T') +_U = TypeVar('_U') + + +def _valueOf(example: _T, cls: Callable[..., _U]) -> Callable[..., _T]: + + def getValue(*args: Any, **kwargs: Any) -> _T: + return cast(_T, cls(*args, **kwargs)) + return getValue + + +class TimeStampParameter(luigi.Parameter): + '''A datetime interchanged as milliseconds since the epoch. + ''' + + def parse(self, s: str) -> datetime: + ms = int(s) + return datetime.fromtimestamp(ms / 1000.0) + + def serialize(self, dt: datetime) -> str: + epoch = datetime.utcfromtimestamp(0) + ms = (dt - epoch).total_seconds() * 1000 + return str(int(ms)) + + +StrParam = _valueOf('s', luigi.Parameter) +IntParam = _valueOf(0, luigi.IntParameter) +BoolParam = _valueOf(True, luigi.BoolParameter) +DictParam = _valueOf({'k': 'v'}, luigi.DictParameter) +TimeStampParam = _valueOf(datetime(2001, 1, 1, 0, 0, 0), TimeStampParameter) diff --git a/pythonjsonlogger/__init__.py b/pythonjsonlogger/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pythonjsonlogger/jsonlogger.py b/pythonjsonlogger/jsonlogger.py new file mode 100644 index 0000000..4f96331 --- /dev/null +++ b/pythonjsonlogger/jsonlogger.py @@ -0,0 +1,141 @@ +''' +This library is provided to allow standard python logging +to output log data as JSON formatted strings +''' +import logging +import json +import re +import datetime +import traceback + +from inspect import istraceback + +#Support order in python 2.7 and 3 +try: + from collections import OrderedDict +except ImportError: + pass + +# skip natural LogRecord attributes +# http://docs.python.org/library/logging.html#logrecord-attributes +RESERVED_ATTRS = ( + 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename', + 'funcName', 'levelname', 'levelno', 'lineno', 'module', + 'msecs', 'message', 'msg', 'name', 'pathname', 'process', + 'processName', 'relativeCreated', 'stack_info', 'thread', 'threadName') + +RESERVED_ATTR_HASH = dict(zip(RESERVED_ATTRS, RESERVED_ATTRS)) + + +def merge_record_extra(record, target, reserved=RESERVED_ATTR_HASH): + """ + Merges extra attributes from LogRecord object into target dictionary + + :param record: logging.LogRecord + :param target: dict to update + :param reserved: dict or list with reserved keys to skip + """ + for key, value in record.__dict__.items(): + #this allows to have numeric keys + if (key not in reserved + and not (hasattr(key, "startswith") + and key.startswith('_'))): + target[key] = value + return target + + +class JsonFormatter(logging.Formatter): + """ + A custom formatter to format logging records as json strings. + extra values will be formatted as str() if nor supported by + json default encoder + """ + + def __init__(self, *args, **kwargs): + """ + :param json_default: a function for encoding non-standard objects + as outlined in http://docs.python.org/2/library/json.html + :param json_encoder: optional custom encoder + :param json_serializer: a :meth:`json.dumps`-compatible callable + that will be used to serialize the log record. + :param prefix: an optional string prefix added at the beginning of + the formatted string + """ + self.json_default = kwargs.pop("json_default", None) + self.json_encoder = kwargs.pop("json_encoder", None) + self.json_serializer = kwargs.pop("json_serializer", json.dumps) + self.prefix = kwargs.pop("prefix", "") + #super(JsonFormatter, self).__init__(*args, **kwargs) + logging.Formatter.__init__(self, *args, **kwargs) + if not self.json_encoder and not self.json_default: + def _default_json_handler(obj): + '''Prints dates in ISO format''' + if isinstance(obj, (datetime.date, datetime.time)): + return obj.isoformat() + elif istraceback(obj): + tb = ''.join(traceback.format_tb(obj)) + return tb.strip() + elif isinstance(obj, Exception): + return "Exception: %s" % str(obj) + return str(obj) + self.json_default = _default_json_handler + self._required_fields = self.parse() + self._skip_fields = dict(zip(self._required_fields, + self._required_fields)) + self._skip_fields.update(RESERVED_ATTR_HASH) + + def parse(self): + """Parses format string looking for substitutions""" + standard_formatters = re.compile(r'\((.+?)\)', re.IGNORECASE) + return standard_formatters.findall(self._fmt) + + def add_fields(self, log_record, record, message_dict): + """ + Override this method to implement custom logic for adding fields. + """ + for field in self._required_fields: + log_record[field] = record.__dict__.get(field) + log_record.update(message_dict) + merge_record_extra(record, log_record, reserved=self._skip_fields) + + def process_log_record(self, log_record): + """ + Override this method to implement custom logic + on the possibly ordered dictionary. + """ + return log_record + + def jsonify_log_record(self, log_record): + """Returns a json string of the log record.""" + return self.json_serializer(log_record, + default=self.json_default, + cls=self.json_encoder) + + def format(self, record): + """Formats a log record and serializes to json""" + message_dict = {} + if isinstance(record.msg, dict): + message_dict = record.msg + record.message = None + else: + record.message = record.getMessage() + # only format time if needed + if "asctime" in self._required_fields: + record.asctime = self.formatTime(record, self.datefmt) + + # Display formatted exception, but allow overriding it in the + # user-supplied dict. + if record.exc_info and not message_dict.get('exc_info'): + message_dict['exc_info'] = self.formatException(record.exc_info) + if not message_dict.get('exc_info') and record.exc_text: + message_dict['exc_info'] = record.exc_text + + try: + log_record = OrderedDict() + except NameError: + log_record = {} + + self.add_fields(log_record, record, message_dict) + log_record = self.process_log_record(log_record) + + return "%s%s" % (self.prefix, self.jsonify_log_record(log_record)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..96cf2a2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,53 @@ +# This is a python packaging requirements file. +# https://packaging.python.org/tutorials/installing-packages/#requirements-files + +# Usage (development): +# +# $ virtualenv --python python3.5 ~/pyenv/grouse3 +# $ . ~/pyenv/grouse3/bin/activate +# (grouse3)$ pip install -r requirements.txt +# ... +# Successfully built luigi tornado +# Installing collected packages: ... +# ... tornado-4.4.2 + + +# Production usage is more like: +# +# $ docker run --rm -ti --entrypoint=/bin/bash stockport/luigi-taskrunner +# app@8ee96e02e548:/$ . /luigi/.pyenv/bin/activate +# (.pyenv) app@8ee96e02e548:/$ python --version +# Python 3.5.2 +# (.pyenv) app@8ee96e02e548:/$ pip freeze +# alembic==0.8.9 + +# cx_Oracle isn't pure-python nor open source, so installation is +# failure-prone. Fortunately, the stockport/luigi-taskrunner Docker +# image provides it: + +cx-Oracle==5.2.1 + +# Cython==0.25.1 +# docutils==0.12 +# lockfile==0.12.2 +luigi==2.4.0 +# Mako==1.0.6 +# MarkupSafe==0.23 +# numpy==1.11.2 +pandas==0.19.1 +# psycopg2==2.6.2 +# pymssql==2.1.3 +# PyPDF2==1.26.0 +# python-daemon==2.1.2 +# python-dateutil==2.6.0 +# python-editor==1.0.3 +# pytz==2016.10 +# requests==2.12.3 +# six==1.10.0 +SQLAlchemy==1.1.4 +#tornado==4.4.2 +#XlsxWriter==0.9.4 + +# let's "vendor" it instead, since +# we have no Internet access from our build jobs. +# python-json-logger==0.1.7 diff --git a/script_lib.py b/script_lib.py new file mode 100644 index 0000000..2297f1b --- /dev/null +++ b/script_lib.py @@ -0,0 +1,306 @@ +r'''script_lib -- library of SQL scripts + +Scripts are pkg_resources, i.e. design-time constants. + +Each script should have a title, taken from the first line:: + + >>> Script.migrate_fact_upload.title + 'append data from a workspace table.' + + >>> text = Script.migrate_fact_upload.value + >>> lines = text.split('\n') + >>> print(lines[0]) + /** migrate_fact_upload - append data from a workspace table. + +We can separate the script into statements:: + + >>> statements = Script.epic_flowsheets_transform.statements() + >>> print(next(s for s in statements if 'insert' in s)) + ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) + with test_key as ( ... + +A bit of sqlplus syntax is supported for ignoring errors in just part +of a script: + + >>> Script.sqlerror('whenever sqlerror exit') + False + >>> Script.sqlerror('whenever sqlerror continue') + True + >>> Script.sqlerror('select 1 + 1 from dual') is None + True + +Dependencies between scripts are declared as follows:: + + >>> print(next(decl for decl in statements if "'dep'" in decl)) + ... #doctest: +ELLIPSIS + select test_name from etl_tests where 'dep' = 'etl_tests_init.sql' + + >>> Script.epic_flowsheets_transform.deps() + ... #doctest: +ELLIPSIS + [] + +We statically detect relevant effects; i.e. tables and views created:: + + >>> Script.epic_flowsheets_transform.created_objects() + ... #doctest: +ELLIPSIS + [view etl_test_domain_flowsheets, view flo_meas_type, view flowsheet_day, ... + +as well as tables inserted into:: + + >>> variables={I2B2STAR: 'I2B2DEMODATA', + ... CMS_RIF: 'CMS_DEID', 'upload_id': '20', 'chunk_qty': 20, + ... 'cms_source_cd': "'ccwdata.org'", 'source_table': 'T'} + >>> Script.epic_flowsheets_transform.inserted_tables(variables) + ['etl_test_values', 'approved_deid_flowsheets', 'etl_test_values'] + +The last statement should be a scalar query that returns non-zero to +signal that the script is complete: + + >>> print(statements[-1]) + ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + select 1 up_to_date + from epic_flowsheets_txform_sql where design_digest = &&design_digest + +The completion test may depend on a digest of the script and its dependencies: + + >>> design_digest = Script.epic_flowsheets_transform.digest() + >>> last = Script.epic_flowsheets_transform.statements(variables)[-1].strip() + >>> print(last) + select 1 up_to_date + from epic_flowsheets_txform_sql where design_digest = 1724122010 + +Some scripts use variables that are not known until a task is run; for +example, `&&upload_id` is used in names of objects such as tables and +partitions; these scripts must not refer to such variables in their +completion query: + + >>> del variables['upload_id'] + >>> print(Script.migrate_fact_upload.statements(variables, + ... skip_unbound=True)[-1].strip()) + commit + +''' + +from itertools import groupby +from typing import Dict, Iterable, List, Optional, Sequence, Text, Tuple, Type +from zlib import adler32 +import enum +import re +import abc + +import pkg_resources as pkg + +import sql_syntax +from sql_syntax import ( + Environment, StatementInContext, ObjectId, SQL, Name, + iter_statement) + +I2B2STAR = 'I2B2STAR' # cf. &&I2B2STAR in sql_scripts +CMS_RIF = 'CMS_RIF' + +ScriptStep = Tuple[int, Text, SQL] +Filename = str + + +class SQLMixin(enum.Enum): + @property + def sql(self) -> SQL: + from typing import cast + return cast(SQL, self.value) # hmm... + + @property + def fname(self) -> str: + return self.name + self.extension + + @abc.abstractproperty + def extension(self) -> str: + raise NotImplementedError + + @abc.abstractmethod + def parse(self, text: SQL) -> Iterable[StatementInContext]: + raise NotImplementedError + + def each_statement(self, + variables: Optional[Environment]=None, + skip_unbound: bool=False) -> Iterable[ScriptStep]: + for line, comment, statement in self.parse(self.sql): + try: + ss = sql_syntax.substitute(statement, self._all_vars(variables)) + except KeyError: + if skip_unbound: + continue + else: + raise + yield line, comment, ss + + def _all_vars(self, variables: Optional[Environment]) -> Optional[Environment]: + '''Add design_digest to variables. + ''' + if variables is None: + return None + return dict(variables, design_digest=str(self.digest())) + + def statements(self, + variables: Optional[Environment]=None, + skip_unbound: bool=False) -> Sequence[Text]: + return list(stmt for _l, _c, stmt + in self.each_statement(skip_unbound=skip_unbound, + variables=variables)) + + def created_objects(self) -> List[ObjectId]: + return [] + + def inserted_tables(self, + variables: Environment={}) -> List[Name]: + return [] + + @property + def title(self) -> Text: + line1 = self.sql.split('\n', 1)[0] + if not (line1.startswith('/** ') and ' - ' in line1): + raise ValueError('%s missing title block' % self) + return line1.split(' - ', 1)[1] + + def deps(self) -> List['SQLMixin']: + return [child + for sql in self.statements() + for child in Script._get_deps(sql)] + + def dep_closure(self) -> List['SQLMixin']: + return [self] + [descendant + for sql in self.statements() + for child in Script._get_deps(sql) + for descendant in child.dep_closure()] + + def digest(self) -> int: + '''Hash the text of this script and its dependencies. + + Unlike the python hash() function, this digest is consistent across runs. + ''' + return adler32(str(self._text()).encode('utf-8')) + + def _text(self) -> List[str]: + '''Get the text of this script and its dependencies. + + >>> nodeps = Script.migrate_fact_upload + >>> nodeps._text() == [nodeps.value] + True + + >>> complex = Script.epic_flowsheets_transform + >>> complex._text() != [complex.value] + True + ''' + return sorted(set(s.sql for s in self.dep_closure())) + + @classmethod + def _get_deps(cls, sql: Text) -> List['SQLMixin']: + ''' + >>> ds = Script._get_deps( + ... "select col from t where 'dep' = 'oops.sql'") + Traceback (most recent call last): + ... + KeyError: 'oops' + + >>> Script._get_deps( + ... "select col from t where x = 'name.sql'") + [] + ''' + from typing import cast + + m = re.search(r"select \S+ from \S+ where 'dep' = '([^']+)'", sql) + if not m: + return [] + name, ext = m.group(1).rsplit('.', 1) + choices = Script if ext == 'sql' else [] + deps = [cast(SQLMixin, s) for s in choices if s.name == name] + if not deps: + raise KeyError(name) + return deps + + @classmethod + def sqlerror(cls, s: SQL) -> Optional[bool]: + if s.strip().lower() == 'whenever sqlerror exit': + return False + elif s.strip().lower() == 'whenever sqlerror continue': + return True + return None + + +class ScriptMixin(SQLMixin): + @property + def extension(self) -> str: + return '.sql' + + def parse(self, text: SQL, + block_sep: str=';\n/\n') -> Iterable[StatementInContext]: + return (sql_syntax.iter_blocks(text) if block_sep in text + else iter_statement(text)) + + def created_objects(self) -> List[ObjectId]: + return [obj + for _l, _comment, stmt in iter_statement(self.sql) + for obj in sql_syntax.created_objects(stmt)] + + def inserted_tables(self, + variables: Optional[Environment]={}) -> List[Name]: + return [obj + for _l, _comment, stmt in iter_statement(self.sql) + for obj in sql_syntax.inserted_tables( + sql_syntax.substitute(stmt, self._all_vars(variables)))] + + +class Script(ScriptMixin, enum.Enum): + '''Script is an enum.Enum of contents. + + ISSUE: It's tempting to consider separate libraries for NAACCR, + NTDS, etc., but that doesn't integrate well with the + generic luigi.EnumParameter() in etl_tasks.SqlScriptTask. + + ''' + [ + # Keep sorted + epic_facts_load, + epic_flowsheets_transform, + etl_tests_init, + migrate_fact_upload, + ] = [ + pkg.resource_string(__name__, + 'sql_scripts/' + fname).decode('utf-8') + for fname in [ + 'epic_facts_load.sql', + 'epic_flowsheets_transform.sql', + 'etl_tests_init.sql', + 'migrate_fact_upload.sql', + ] + ] + + def __repr__(self) -> str: + return '<%s(%s)>' % (self.__class__.__name__, self.name) + + +def _object_to_creators(libs: List[Type[SQLMixin]]) -> Dict[ObjectId, List[SQLMixin]]: + '''Find creator scripts for each object. + + "There can be only one." + >>> creators = _object_to_creators([Script]) + >>> [obj for obj, scripts in creators.items() + ... if len(scripts) > 1] + [] + ''' + fst = lambda pair: pair[0] + snd = lambda pair: pair[1] + + objs = sorted( + [(obj, s) + for lib in libs for s in lib + for obj in s.created_objects()], + key=fst) + by_obj = groupby(objs, key=fst) + return dict((obj, list(map(snd, places))) for obj, places in by_obj) + + +_redefined_objects = [ + obj for obj, scripts in _object_to_creators([Script]).items() + if len(scripts) > 1] +assert _redefined_objects == [], _redefined_objects diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3c3a387 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,49 @@ +# Usage: +# +# $ nosetests && flake8 . && mypy . + + +[nosetests] +# verbosity=3 +with-doctest=1 +where=. + +[flake8] +# E731: Assigning to lambda seems OK +# E126: Emacs python mode seems to over-indent? +ignore = E731, E126 +# with type annotations, 79 is awkward +# guide, for window sizing: +# 23456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 +max-line-length = 120 +exclude = pythonjsonlogger + + +[mypy] +# http://mypy.readthedocs.io/en/latest/config_file.html +mypy_path=stubs +warn_redundant_casts=true +warn_unused_ignores=true +strict_optional=true + +strict_boolean=false +# but all the other --strict flags (from mypy -h): +disallow_untyped_calls=true +disallow_untyped_defs=true +check_untyped_defs=true +warn_return_any=true + +[mypy-luigi.*,cx_Oracle.*,sqlalchemy.*] +disallow_untyped_defs=false + +[mypy-spreadsync.*] +ignore_errors = True + +[mypy-sqlalchemy.*] +ignore_errors = True + +[mypy-pythonjsonlogger.*] +ignore_errors = True + +#--cobertura-xml-report DIR +#--junit-xml JUNIT_XML diff --git a/sql-style.xml b/sql-style.xml new file mode 100644 index 0000000..6e1e56a --- /dev/null +++ b/sql-style.xml @@ -0,0 +1,453 @@ + + + + + 1:kumc-bmi-sql-style + + false + false + false + true + true + true + false + false + 1 + true + false + false + false + true + true + false + true + true + true + true + false + true + true + false + 2 + false + false + 3 + false + false + true + 10 + 120 + + false + kumc-bmi-sql-style + 1 + 2 + 2 + 2 + 0 + false + 60 + 1 + 0 + 0 + false + false + 0 + 3 + false + + + + 1:Old Preferences + + true + false + true + true + true + true + false + true + 0 + false + false + true + false + true + false + false + true + true + true + true + true + true + true + false + 0 + false + false + 0 + false + true + false + 999 + 999 + + false + Old Preferences + 1 + 2 + 0 + 0 + 0 + false + 80 + 1 + 0 + 0 + false + false + 0 + 0 + false + + + + 1:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 1 + 0 + 0 + false + false + 0 + 0 + false + + + + 2:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 2 + 0 + 0 + false + false + 0 + 0 + false + + + + 3:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 3 + 0 + 0 + false + false + 0 + 0 + false + + + + 4:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 4 + 0 + 0 + false + false + 0 + 0 + false + + + + 5:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 5 + 0 + 0 + false + false + 0 + 0 + false + + + + 6:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 6 + 0 + 0 + false + false + 0 + 0 + false + + + + diff --git a/sql_syntax.py b/sql_syntax.py new file mode 100644 index 0000000..6e00e94 --- /dev/null +++ b/sql_syntax.py @@ -0,0 +1,238 @@ +'''sql_syntax - break SQL scripts into statements, etc. + +''' + +from datetime import datetime +from typing import Dict, Iterable, List, Optional, Text, Tuple, Union +import re + +Name = Text +SQL = Text +Environment = Dict[Name, Text] +Params = Dict[str, Union[str, int, datetime]] +Line = int +Comment = Text +StatementInContext = Tuple[Line, Comment, SQL] + + +def iter_statement(txt: SQL) -> Iterable[StatementInContext]: + r'''Iterate over SQL statements in a script. + + >>> list(iter_statement("drop table foo; create table foo")) + [(1, '', 'drop table foo'), (1, '', 'create table foo')] + + >>> list(iter_statement("-- blah blah\ndrop table foo")) + [(2, '-- blah blah\n', 'drop table foo')] + + >>> list(iter_statement("drop /* blah blah */ table foo")) + [(1, '', 'drop table foo')] + + >>> list(iter_statement("select '[^;]+' from dual")) + [(1, '', "select '[^;]+' from dual")] + + >>> list(iter_statement('select "x--y" from z;')) + [(1, '', 'select "x--y" from z')] + >>> list(iter_statement("select 'x--y' from z;")) + [(1, '', "select 'x--y' from z")] + ''' + + statement = comment = '' + line = 1 + sline = None # type: Optional[int] + + def save(txt: SQL) -> Tuple[SQL, Optional[int]]: + return (statement + txt, sline or (line if txt else None)) + + while 1: + m = SQL_SEPARATORS.search(txt) + if not m: + statement, sline = save(txt) + break + + pfx, match, txt = (txt[:m.start()], + txt[m.start():m.end()], + txt[m.end():]) + if pfx: + statement, sline = save(pfx) + + if m.group('sep'): + if sline: + yield sline, comment, statement + statement = comment = '' + sline = None + elif [n for n in ('lit', 'hint', 'sym') + if m.group(n)]: + statement, sline = save(match) + elif (m.group('space') and statement): + statement, sline = save(match) + elif ((m.group('comment') and not statement) or + (m.group('space') and comment)): + comment += match + + line += (pfx + match).count("\n") + + if sline and (comment or statement): + yield sline, comment, statement + + +# Check for hint before comment since a hint looks like a comment +SQL_SEPARATORS = re.compile( + r'(?P^\s+)' + r'|(?P/\*\+.*?\*/)' + r'|(?P(--[^\n]*(?:\n|$))|(?:/\*([^\*]|(\*(?!/)))*\*/))' + r'|(?P"[^\"]*")' + r"|(?P'[^\']*')" + r'|(?P;)') + + +def _test_iter_statement() -> None: + r''' + >>> list(iter_statement("/* blah blah */ drop table foo")) + [(1, '/* blah blah */ ', 'drop table foo')] + + >>> [l for (l, c, s) in iter_statement('s1;\n/**********************\n' + ... '* Medication concepts \n' + ... '**********************/\ns2')] + [1, 5] + + >>> list(iter_statement("")) + [] + + >>> list(iter_statement("/*...*/ ")) + [] + + >>> list(iter_statement("drop table foo; ")) + [(1, '', 'drop table foo')] + + >>> list(iter_statement("drop table foo")) + [(1, '', 'drop table foo')] + + >>> list(iter_statement("/* *** -- *** */ drop table foo")) + [(1, '/* *** -- *** */ ', 'drop table foo')] + + >>> list(iter_statement("/* *** -- *** */ drop table x /* ... */")) + [(1, '/* *** -- *** */ ', 'drop table x ')] + + >>> list(iter_statement('select /*+ index(ix1 ix2) */ * from some_table')) + [(1, '', 'select /*+ index(ix1 ix2) */ * from some_table')] + + >>> len(list(iter_statement(""" /* + no space before + in hints*/ + ... /*+ index(observation_fact fact_cnpt_pat_enct_idx) */ + ... select * from some_table; """))[0][2]) + 79 + + >>> list(iter_statement('select 1+1; /* nothing else */; ')) + [(1, '', 'select 1+1')] + ''' + pass # pragma: nocover + + +def substitute(sql: SQL, variables: Optional[Environment]) -> SQL: + '''Evaluate substitution variables in the style of Oracle sqlplus. + + >>> substitute('select &¬_bound from dual', {}) + Traceback (most recent call last): + KeyError: 'not_bound' + ''' + if variables is None: + return sql + sql_esc = sql.replace('%', '%%') # escape %, which we use specially + return re.sub('&&(\w+)', r'%(\1)s', sql_esc) % variables + + +def params_used(params: Params, statement: SQL) -> Params: + return dict((k, v) for (k, v) in params.items() + if k in param_names(statement)) + + +def param_names(s: SQL) -> List[Name]: + ''' + >>> param_names('select 1+1 from dual') + [] + >>> param_names('select 1+:y from dual') + ['y'] + ''' + return [expr[1:] + for expr in re.findall(r':\w+', s)] + + +def first_cursor(statement: SQL) -> SQL: + '''Find argument of first obvious call to `cursor()`. + + >>> first_cursor('select * from table(f(cursor(select * from there)))') + 'select * from there' + ''' + return statement.split('cursor(')[1].split(')')[0] + + +class ObjectId(object): + kind = '' + + def __init__(self, name: str) -> None: + self.name = name + + def __repr__(self) -> str: + return '{} {}'.format(self.kind, self.name) + + def __hash__(self) -> int: + return hash((self.kind, self.name)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ObjectId): + return False + return (self.kind, self.name) == ((other.kind, other.name)) + + def __lt__(self, other: 'ObjectId') -> bool: + return (self.kind, self.name) < ((other.kind, other.name)) + + +class TableId(ObjectId): + kind = 'table' + + +class ViewId(ObjectId): + kind = 'view' + + +def created_objects(statement: SQL) -> List[ObjectId]: + r''' + >>> created_objects('create table t as ...') + [table t] + + >>> created_objects('create or replace view x\nas ...') + [view x] + ''' + m = re.search('^create or replace view (\S+)', statement.strip()) + views = [ViewId(m.group(1))] if m else [] # type: List[ObjectId] + m = re.search('^create table (\S+)', statement.strip()) + tables = [TableId(m.group(1))] if m else [] # type: List[ObjectId] + return tables + views + + +def inserted_tables(statement: SQL) -> List[Name]: + r''' + >>> inserted_tables('create table t as ...') + [] + >>> inserted_tables('insert into t (...) ...') + ['t'] + ''' + if not statement.startswith('insert'): + return [] + m = re.search('into\s+(\S+)', statement.strip()) + return [m.group(1)] if m else [] + + +def insert_append_table(statement: SQL) -> Optional[Name]: + if '/*+ append' in statement: + [t] = inserted_tables(statement) + return t + return None + + +def iter_blocks(module: SQL, + separator: str ='\n/\n') -> Iterable[StatementInContext]: + line = 1 + for block in module.split(separator): + if block.strip(): + yield (line, '...no comment handling...', block) + line += len((block + separator).split('\n')[:-1]) diff --git a/stubs/cx_Oracle.pyi b/stubs/cx_Oracle.pyi new file mode 100644 index 0000000..10a8948 --- /dev/null +++ b/stubs/cx_Oracle.pyi @@ -0,0 +1,429 @@ +# Stubs for cx_Oracle (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from datetime import date +from typing import Any + +ATTR_PURITY_DEFAULT = ... # type: int +ATTR_PURITY_NEW = ... # type: int +ATTR_PURITY_SELF = ... # type: int +DBSHUTDOWN_ABORT = ... # type: int +DBSHUTDOWN_FINAL = ... # type: int +DBSHUTDOWN_IMMEDIATE = ... # type: int +DBSHUTDOWN_TRANSACTIONAL = ... # type: int +DBSHUTDOWN_TRANSACTIONAL_LOCAL = ... # type: int +EVENT_DEREG = ... # type: int +EVENT_NONE = ... # type: int +EVENT_OBJCHANGE = ... # type: int +EVENT_QUERYCHANGE = ... # type: int +EVENT_SHUTDOWN = ... # type: int +EVENT_SHUTDOWN_ANY = ... # type: int +EVENT_STARTUP = ... # type: int +FNCODE_BINDBYNAME = ... # type: int +FNCODE_BINDBYPOS = ... # type: int +FNCODE_DEFINEBYPOS = ... # type: int +FNCODE_STMTEXECUTE = ... # type: int +FNCODE_STMTFETCH = ... # type: int +FNCODE_STMTPREPARE = ... # type: int +OPCODE_ALLOPS = ... # type: int +OPCODE_ALLROWS = ... # type: int +OPCODE_ALTER = ... # type: int +OPCODE_DELETE = ... # type: int +OPCODE_DROP = ... # type: int +OPCODE_INSERT = ... # type: int +OPCODE_UPDATE = ... # type: int +PRELIM_AUTH = ... # type: int +SPOOL_ATTRVAL_FORCEGET = ... # type: int +SPOOL_ATTRVAL_NOWAIT = ... # type: int +SPOOL_ATTRVAL_WAIT = ... # type: int +SUBSCR_CQ_QOS_BEST_EFFORT = ... # type: int +SUBSCR_CQ_QOS_CLQRYCACHE = ... # type: int +SUBSCR_CQ_QOS_QUERY = ... # type: int +SUBSCR_NAMESPACE_DBCHANGE = ... # type: int +SUBSCR_PROTO_HTTP = ... # type: int +SUBSCR_PROTO_MAIL = ... # type: int +SUBSCR_PROTO_OCI = ... # type: int +SUBSCR_PROTO_SERVER = ... # type: int +SUBSCR_QOS_HAREG = ... # type: int +SUBSCR_QOS_MULTICBK = ... # type: int +SUBSCR_QOS_PAYLOAD = ... # type: int +SUBSCR_QOS_PURGE_ON_NTFN = ... # type: int +SUBSCR_QOS_RELIABLE = ... # type: int +SUBSCR_QOS_REPLICATE = ... # type: int +SUBSCR_QOS_SECURE = ... # type: int +SYSASM = ... # type: int +SYSDBA = ... # type: int +SYSOPER = ... # type: int +UCBTYPE_ENTRY = ... # type: int +UCBTYPE_EXIT = ... # type: int +UCBTYPE_REPLACE = ... # type: int +apilevel = ... # type: str +buildtime = ... # type: str +paramstyle = ... # type: str +threadsafety = ... # type: int +version = ... # type: str + +def DateFromTicks(*args, **kwargs): ... +def Time(*args, **kwargs): ... +def TimeFromTicks(*args, **kwargs): ... +def TimestampFromTicks(*args, **kwargs): ... +def clientversion(*args, **kwargs): ... +def makedsn(*args, **kwargs): ... + +class _BASEVARTYPE(object): ... + +class BFILE(_BASEVARTYPE): ... + +class BINARY(_BASEVARTYPE): ... + +class BLOB(_BASEVARTYPE): ... + +class Binary: + maketrans = ... # type: Any + def __init__(self, *args, **kwargs): ... + def capitalize(self, *args, **kwargs): ... + def center(self, *args, **kwargs): ... + def count(self, *args, **kwargs): ... + def decode(self, *args, **kwargs): ... + def endswith(self, *args, **kwargs): ... + def expandtabs(self, *args, **kwargs): ... + def find(self, *args, **kwargs): ... + @classmethod + def fromhex(cls, *args, **kwargs): ... + def hex(self, *args, **kwargs): ... + def index(self, *args, **kwargs): ... + def isalnum(self, *args, **kwargs): ... + def isalpha(self, *args, **kwargs): ... + def isdigit(self, *args, **kwargs): ... + def islower(self, *args, **kwargs): ... + def isspace(self, *args, **kwargs): ... + def istitle(self, *args, **kwargs): ... + def isupper(self, *args, **kwargs): ... + def join(self, *args, **kwargs): ... + def ljust(self, *args, **kwargs): ... + def lower(self, *args, **kwargs): ... + def lstrip(self, *args, **kwargs): ... + def partition(self, *args, **kwargs): ... + def replace(self, *args, **kwargs): ... + def rfind(self, *args, **kwargs): ... + def rindex(self, *args, **kwargs): ... + def rjust(self, *args, **kwargs): ... + def rpartition(self, *args, **kwargs): ... + def rsplit(self, *args, **kwargs): ... + def rstrip(self, *args, **kwargs): ... + def split(self, *args, **kwargs): ... + def splitlines(self, *args, **kwargs): ... + def startswith(self, *args, **kwargs): ... + def strip(self, *args, **kwargs): ... + def swapcase(self, *args, **kwargs): ... + def title(self, *args, **kwargs): ... + def translate(self, *args, **kwargs): ... + def upper(self, *args, **kwargs): ... + def zfill(self, *args, **kwargs): ... + def __add__(self, other): ... + def __contains__(self, *args, **kwargs): ... + def __eq__(self, other): ... + def __ge__(self, other): ... + def __getitem__(self, index): ... + def __getnewargs__(self, *args, **kwargs): ... + def __gt__(self, other): ... + def __hash__(self): ... + def __iter__(self): ... + def __le__(self, other): ... + def __len__(self, *args, **kwargs): ... + def __lt__(self, other): ... + def __mod__(self, other): ... + def __mul__(self, other): ... + def __ne__(self, other): ... + def __rmod__(self, other): ... + def __rmul__(self, other): ... + +class CLOB(_BASEVARTYPE): ... + +class CURSOR(_BASEVARTYPE): ... + +class Connection: + action = ... # type: Any + autocommit = ... # type: Any + client_identifier = ... # type: Any + clientinfo = ... # type: Any + current_schema = ... # type: Any + dsn = ... # type: Any + encoding = ... # type: Any + inputtypehandler = ... # type: Any + maxBytesPerCharacter = ... # type: Any + module = ... # type: Any + nencoding = ... # type: Any + outputtypehandler = ... # type: Any + stmtcachesize = ... # type: Any + tnsentry = ... # type: Any + username = ... # type: Any + version = ... # type: Any + def __init__(self, *args, **kwargs): ... + def begin(self, *args, **kwargs): ... + def cancel(self, *args, **kwargs): ... + def changepassword(self, *args, **kwargs): ... + def close(self, *args, **kwargs): ... + def commit(self, *args, **kwargs): ... + def cursor(self, *args, **kwargs): ... + def ping(self, *args, **kwargs): ... + def prepare(self, *args, **kwargs): ... + def register(self, *args, **kwargs): ... + def rollback(self, *args, **kwargs): ... + def shutdown(self, *args, **kwargs): ... + def startup(self, *args, **kwargs): ... + def subscribe(self, *args, **kwargs): ... + def unregister(self, *args, **kwargs): ... + def __enter__(self, *args, **kwargs): ... + def __exit__(self, *args, **kwargs): ... + +class Cursor: + arraysize = ... # type: Any + bindarraysize = ... # type: Any + bindvars = ... # type: Any + connection = ... # type: Any + description = ... # type: Any + fetchvars = ... # type: Any + inputtypehandler = ... # type: Any + numbersAsStrings = ... # type: Any + outputtypehandler = ... # type: Any + rowcount = ... # type: Any + rowfactory = ... # type: Any + statement = ... # type: Any + def __init__(self, *args, **kwargs): ... + def arrayvar(self, *args, **kwargs): ... + def bindnames(self, *args, **kwargs): ... + def callfunc(self, *args, **kwargs): ... + def callproc(self, *args, **kwargs): ... + def close(self, *args, **kwargs): ... + def execute(self, *args, **kwargs): ... + def executemany(self, *args, **kwargs): ... + def executemanyprepared(self, *args, **kwargs): ... + def fetchall(self, *args, **kwargs): ... + def fetchmany(self, *args, **kwargs): ... + def fetchone(self, *args, **kwargs): ... + def fetchraw(self, *args, **kwargs): ... + def getbatcherrors(self, *args, **kwargs): ... + def parse(self, *args, **kwargs): ... + def prepare(self, *args, **kwargs): ... + def setinputsizes(self, *args, **kwargs): ... + def setoutputsize(self, *args, **kwargs): ... + def var(self, *args, **kwargs): ... + def __iter__(self): ... + def __next__(self): ... + +class DATETIME(_BASEVARTYPE): ... + +class DataError(DatabaseError): ... + +class DatabaseError(Error): ... + +class Date: + day = ... # type: Any + max = ... # type: Any + min = ... # type: Any + month = ... # type: Any + resolution = ... # type: Any + year = ... # type: Any + def __init__(self, *args, **kwargs): ... + def ctime(self, *args, **kwargs): ... + @classmethod + def fromordinal(cls, *args, **kwargs): ... + @classmethod + def fromtimestamp(cls, *args, **kwargs): ... + def isocalendar(self, *args, **kwargs): ... + def isoformat(self, *args, **kwargs): ... + def isoweekday(self, *args, **kwargs): ... + def replace(self, *args, **kwargs): ... + def strftime(self, *args, **kwargs): ... + def timetuple(self, *args, **kwargs): ... + @classmethod + def today(cls, *args, **kwargs): ... + def toordinal(self, *args, **kwargs): ... + def weekday(self, *args, **kwargs): ... + def __add__(self, other): ... + def __eq__(self, other): ... + def __format__(self, *args, **kwargs): ... + def __ge__(self, other): ... + def __gt__(self, other): ... + def __hash__(self): ... + def __le__(self, other): ... + def __lt__(self, other): ... + def __ne__(self, other): ... + def __radd__(self, other): ... + def __reduce__(self): ... + def __rsub__(self, other): ... + def __sub__(self, other): ... + +class Error(Exception): ... + +class FIXED_CHAR(_BASEVARTYPE): ... + +class FIXED_NCHAR(_BASEVARTYPE): ... + +class FIXED_UNICODE(_BASEVARTYPE): ... + +class INTERVAL(_BASEVARTYPE): ... + +class IntegrityError(DatabaseError): ... + +class InterfaceError(Error): ... + +class InternalError(DatabaseError): ... + +class LOB: + def close(self, *args, **kwargs): ... + def fileexists(self, *args, **kwargs): ... + def getchunksize(self, *args, **kwargs): ... + def getfilename(self, *args, **kwargs): ... + def isopen(self, *args, **kwargs): ... + def open(self, *args, **kwargs): ... + def read(self, *args, **kwargs): ... + def setfilename(self, *args, **kwargs): ... + def size(self, *args, **kwargs): ... + def trim(self, *args, **kwargs): ... + def write(self, *args, **kwargs): ... + def __reduce__(self): ... + +class LONG_BINARY(_BASEVARTYPE): ... + +class LONG_NCHAR(_BASEVARTYPE): ... + +class LONG_STRING(_BASEVARTYPE): ... + +class LONG_UNICODE(_BASEVARTYPE): ... + +class NATIVE_FLOAT(_BASEVARTYPE): ... + +class NCHAR(_BASEVARTYPE): ... + +class NCLOB(_BASEVARTYPE): ... + +class NUMBER(_BASEVARTYPE): ... + +class NotSupportedError(DatabaseError): ... + +class OBJECT(_BASEVARTYPE): + type = ... # type: Any + +class OperationalError(DatabaseError): ... + +class ProgrammingError(DatabaseError): ... + +class ROWID(_BASEVARTYPE): ... + +class STRING(_BASEVARTYPE): ... + +class SessionPool: + busy = ... # type: Any + dsn = ... # type: Any + getmode = ... # type: Any + homogeneous = ... # type: Any + increment = ... # type: Any + max = ... # type: Any + min = ... # type: Any + name = ... # type: Any + opened = ... # type: Any + timeout = ... # type: Any + tnsentry = ... # type: Any + username = ... # type: Any + def __init__(self, *args, **kwargs): ... + def acquire(self, *args, **kwargs): ... + def drop(self, *args, **kwargs): ... + def release(self, *args, **kwargs): ... + +class TIMESTAMP(_BASEVARTYPE): ... + +class Timestamp(date): + hour = ... # type: Any + max = ... # type: Any + microsecond = ... # type: Any + min = ... # type: Any + minute = ... # type: Any + resolution = ... # type: Any + second = ... # type: Any + tzinfo = ... # type: Any + def __init__(self, *args, **kwargs): ... + def astimezone(self, *args, **kwargs): ... + @classmethod + def combine(cls, *args, **kwargs): ... + def ctime(self, *args, **kwargs): ... + def date(self, *args, **kwargs): ... + def dst(self, *args, **kwargs): ... + @classmethod + def fromtimestamp(cls, *args, **kwargs): ... + def isoformat(self, *args, **kwargs): ... + @classmethod + def now(cls, *args, **kwargs): ... + def replace(self, *args, **kwargs): ... + @classmethod + def strptime(cls, *args, **kwargs): ... + def time(self, *args, **kwargs): ... + def timestamp(self, *args, **kwargs): ... + def timetuple(self, *args, **kwargs): ... + def timetz(self, *args, **kwargs): ... + def tzname(self, *args, **kwargs): ... + @classmethod + def utcfromtimestamp(cls, *args, **kwargs): ... + @classmethod + def utcnow(cls, *args, **kwargs): ... + def utcoffset(self, *args, **kwargs): ... + def utctimetuple(self, *args, **kwargs): ... + def __add__(self, other): ... + def __eq__(self, other): ... + def __ge__(self, other): ... + def __gt__(self, other): ... + def __hash__(self): ... + def __le__(self, other): ... + def __lt__(self, other): ... + def __ne__(self, other): ... + def __radd__(self, other): ... + def __reduce__(self): ... + def __rsub__(self, other): ... + def __sub__(self, other): ... + +class UNICODE(_BASEVARTYPE): ... + +class Warning(Exception): ... + +class _Error: + code = ... # type: Any + context = ... # type: Any + message = ... # type: Any + offset = ... # type: Any + +class connect: + action = ... # type: Any + autocommit = ... # type: Any + client_identifier = ... # type: Any + clientinfo = ... # type: Any + current_schema = ... # type: Any + dsn = ... # type: Any + encoding = ... # type: Any + inputtypehandler = ... # type: Any + maxBytesPerCharacter = ... # type: Any + module = ... # type: Any + nencoding = ... # type: Any + outputtypehandler = ... # type: Any + stmtcachesize = ... # type: Any + tnsentry = ... # type: Any + username = ... # type: Any + version = ... # type: Any + def __init__(self, *args, **kwargs): ... + def begin(self, *args, **kwargs): ... + def cancel(self, *args, **kwargs): ... + def changepassword(self, *args, **kwargs): ... + def close(self, *args, **kwargs): ... + def commit(self, *args, **kwargs): ... + def cursor(self, *args, **kwargs): ... + def ping(self, *args, **kwargs): ... + def prepare(self, *args, **kwargs): ... + def register(self, *args, **kwargs): ... + def rollback(self, *args, **kwargs): ... + def shutdown(self, *args, **kwargs): ... + def startup(self, *args, **kwargs): ... + def subscribe(self, *args, **kwargs): ... + def unregister(self, *args, **kwargs): ... + def __enter__(self, *args, **kwargs): ... + def __exit__(self, *args, **kwargs): ... diff --git a/stubs/luigi/__init__.pyi b/stubs/luigi/__init__.pyi new file mode 100644 index 0000000..3a21420 --- /dev/null +++ b/stubs/luigi/__init__.pyi @@ -0,0 +1,20 @@ +# Stubs for luigi (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from luigi import task as task +from luigi.task import Task as Task, Config as Config, ExternalTask as ExternalTask, WrapperTask as WrapperTask, namespace as namespace, auto_namespace as auto_namespace +from luigi import target as target +from luigi.target import Target as Target +from luigi import local_target as local_target +from luigi.local_target import LocalTarget as LocalTarget +from luigi import rpc as rpc +from luigi.rpc import RemoteScheduler as RemoteScheduler, RPCError as RPCError +from luigi import parameter as parameter +from luigi.parameter import Parameter as Parameter, DateParameter as DateParameter, MonthParameter as MonthParameter, YearParameter as YearParameter, DateHourParameter as DateHourParameter, DateMinuteParameter as DateMinuteParameter, DateSecondParameter as DateSecondParameter, DateIntervalParameter as DateIntervalParameter, TimeDeltaParameter as TimeDeltaParameter, IntParameter as IntParameter, FloatParameter as FloatParameter, BoolParameter as BoolParameter, TaskParameter as TaskParameter, EnumParameter as EnumParameter, DictParameter as DictParameter, ListParameter as ListParameter, TupleParameter as TupleParameter, NumericalParameter as NumericalParameter, ChoiceParameter as ChoiceParameter +from luigi import configuration as configuration +from luigi import interface as interface +from luigi.interface import run as run, build as build +from luigi import event as event +from luigi.event import Event as Event +from .tools import range as range diff --git a/stubs/luigi/configuration.pyi b/stubs/luigi/configuration.pyi new file mode 100644 index 0000000..06a65a2 --- /dev/null +++ b/stubs/luigi/configuration.pyi @@ -0,0 +1,26 @@ +# Stubs for luigi.configuration (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional +# py2: from ConfigParser import ConfigParser +from configparser import ConfigParser + +class LuigiConfigParser(ConfigParser): + NO_DEFAULT = ... # type: Any + config_file = ... # type: Any + @classmethod + def add_config_path(cls, path): ... + @classmethod + def instance(cls, *args, **kwargs): ... + @classmethod + def reload(cls): ... + # avoid: Signature of "get" incompatible with supertype "Mapping" + def get(self, section, option, default: Any = ..., **kwargs): ... # type:ignore + def getboolean(self, section, option, default: Any = ...): ... # type:ignore + def getint(self, section, option, default: Any = ...): ... # type:ignore + def getfloat(self, section, option, default: Any = ...): ... # type:ignore + def getintdict(self, section): ... + def set(self, section, option, value: Optional[Any] = ...): ... + +def get_config(): ... diff --git a/stubs/luigi/contrib/__init__.pyi b/stubs/luigi/contrib/__init__.pyi new file mode 100644 index 0000000..1a7b73e --- /dev/null +++ b/stubs/luigi/contrib/__init__.pyi @@ -0,0 +1,4 @@ +# Stubs for luigi.contrib (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + diff --git a/stubs/luigi/contrib/sqla.pyi b/stubs/luigi/contrib/sqla.pyi new file mode 100644 index 0000000..eb90250 --- /dev/null +++ b/stubs/luigi/contrib/sqla.pyi @@ -0,0 +1,43 @@ +# Stubs for luigi.contrib.sqla (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Dict, Any +import luigi +from collections import namedtuple +from sqlalchemy.engine import Engine + +class SQLAlchemyTarget(luigi.Target): + marker_table = ... # type: Any + + Connection = namedtuple('Connection', 'engine pid') + target_table = ... # type: str + update_id = ... # type: str + connection_string = ... # type: str + echo = ... # type: bool + connect_args = ... # type: Dict[str, Any] + marker_table_bound = ... # type: Any + def __init__(self, connection_string, target_table, update_id, echo: bool = ..., connect_args: Any = ...) -> None: ... + @property + def engine(self) -> Engine: ... + def touch(self) -> None: ... + def exists(self) -> bool: ... + def create_marker_table(self) -> None: ... + def open(self, mode): ... + +class CopyToTable(luigi.Task): + echo = ... # type: bool + connect_args = ... # type: Any + def connection_string(self): ... + def table(self): ... + columns = ... # type: Any + column_separator = ... # type: str + chunk_size = ... # type: int + reflect = ... # type: bool + table_bound = ... # type: Any + def create_table(self, engine): ... + def update_id(self): ... + def output(self): ... + def rows(self): ... + def run(self): ... + def copy(self, conn, ins_rows, table_bound): ... diff --git a/stubs/luigi/event.pyi b/stubs/luigi/event.pyi new file mode 100644 index 0000000..1cfa689 --- /dev/null +++ b/stubs/luigi/event.pyi @@ -0,0 +1,15 @@ +# Stubs for luigi.event (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +class Event: + DEPENDENCY_DISCOVERED = ... # type: str + DEPENDENCY_MISSING = ... # type: str + DEPENDENCY_PRESENT = ... # type: str + BROKEN_TASK = ... # type: str + START = ... # type: str + FAILURE = ... # type: str + SUCCESS = ... # type: str + PROCESSING_TIME = ... # type: str + TIMEOUT = ... # type: str + PROCESS_FAILURE = ... # type: str diff --git a/stubs/luigi/interface.pyi b/stubs/luigi/interface.pyi new file mode 100644 index 0000000..950c658 --- /dev/null +++ b/stubs/luigi/interface.pyi @@ -0,0 +1,37 @@ +# Stubs for luigi.interface (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional +from . import task + +def setup_interface_logging(conf_file: str = ..., level_name: str = ...): ... + +class core(task.Config): + use_cmdline_section = ... # type: bool + local_scheduler = ... # type: Any + scheduler_host = ... # type: Any + scheduler_port = ... # type: Any + scheduler_url = ... # type: Any + lock_size = ... # type: Any + no_lock = ... # type: Any + lock_pid_dir = ... # type: Any + take_lock = ... # type: Any + workers = ... # type: Any + logging_conf_file = ... # type: Any + log_level = ... # type: Any + module = ... # type: Any + parallel_scheduling = ... # type: Any + assistant = ... # type: Any + help = ... # type: Any + help_all = ... # type: Any + +class _WorkerSchedulerFactory: + def create_local_scheduler(self): ... + def create_remote_scheduler(self, url): ... + def create_worker(self, scheduler, worker_processes, assistant: bool = ...): ... + +class PidLockAlreadyTakenExit(SystemExit): ... + +def run(*args, **kwargs): ... +def build(tasks, worker_scheduler_factory: Optional[Any] = ..., **env_params): ... diff --git a/stubs/luigi/local_target.pyi b/stubs/luigi/local_target.pyi new file mode 100644 index 0000000..d3883e4 --- /dev/null +++ b/stubs/luigi/local_target.pyi @@ -0,0 +1,35 @@ +# Stubs for luigi.local_target (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional +from luigi.target import FileSystem, FileSystemTarget, AtomicLocalFile + +class atomic_file(AtomicLocalFile): + def move_to_final_destination(self): ... + def generate_tmp_path(self, path): ... + +class LocalFileSystem(FileSystem): + def copy(self, old_path, new_path, raise_if_exists: bool = ...): ... + def exists(self, path): ... + def mkdir(self, path, parents: bool = ..., raise_if_exists: bool = ...): ... + def isdir(self, path): ... + def listdir(self, path): ... + # Signature of "remove" incompatible with supertype "FileSystem" + def remove(self, path, recursive: bool = ...): ... # type: ignore + def move(self, old_path, new_path, raise_if_exists: bool = ...): ... + +class LocalTarget(FileSystemTarget): + fs = ... # type: Any + format = ... # type: Any + is_tmp = ... # type: Any + def __init__(self, path: Optional[Any] = ..., format: Optional[Any] = ..., is_tmp: bool = ...) -> None: ... + def makedirs(self): ... + def open(self, mode: str = ...): ... + def move(self, new_path, raise_if_exists: bool = ...): ... + def move_dir(self, new_path): ... + def remove(self): ... + def copy(self, new_path, raise_if_exists: bool = ...): ... + @property + def fn(self): ... + def __del__(self): ... diff --git a/stubs/luigi/parameter.pyi b/stubs/luigi/parameter.pyi new file mode 100644 index 0000000..a550034 --- /dev/null +++ b/stubs/luigi/parameter.pyi @@ -0,0 +1,131 @@ +# Stubs for luigi.parameter (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from json.encoder import JSONEncoder +from typing import Any, Optional +from collections import Mapping + +class ParameterException(Exception): ... +class MissingParameterException(ParameterException): ... +class UnknownParameterException(ParameterException): ... +class DuplicateParameterException(ParameterException): ... + +class Parameter: + significant = ... # type: Any + positional = ... # type: Any + description = ... # type: Any + always_in_help = ... # type: Any + def __init__(self, default: Any = ..., is_global: bool = ..., significant: bool = ..., description: Optional[Any] = ..., config_path: Optional[Any] = ..., positional: bool = ..., always_in_help: bool = ..., batch_method: Optional[Any] = ...) -> None: ... + def has_task_value(self, task_name, param_name): ... + def task_value(self, task_name, param_name): ... + def parse(self, x): ... + def serialize(self, x): ... + def normalize(self, x): ... + def next_in_enumeration(self, _value): ... + +class _DateParameterBase(Parameter): + interval = ... # type: Any + start = ... # type: Any + def __init__(self, interval: int = ..., start: Optional[Any] = ..., **kwargs) -> None: ... + def date_format(self): ... + def parse(self, s): ... + def serialize(self, dt): ... + +class DateParameter(_DateParameterBase): + date_format = ... # type: str + def next_in_enumeration(self, value): ... + def normalize(self, value): ... + +class MonthParameter(DateParameter): + date_format = ... # type: str + def next_in_enumeration(self, value): ... + def normalize(self, value): ... + +class YearParameter(DateParameter): + date_format = ... # type: str + def next_in_enumeration(self, value): ... + def normalize(self, value): ... + +class _DatetimeParameterBase(Parameter): + interval = ... # type: Any + start = ... # type: Any + def __init__(self, interval: int = ..., start: Optional[Any] = ..., **kwargs) -> None: ... + def date_format(self): ... + def parse(self, s): ... + def serialize(self, dt): ... + def normalize(self, dt): ... + def next_in_enumeration(self, value): ... + +class DateHourParameter(_DatetimeParameterBase): + date_format = ... # type: str + +class DateMinuteParameter(_DatetimeParameterBase): + date_format = ... # type: str + deprecated_date_format = ... # type: str + def parse(self, s): ... + +class DateSecondParameter(_DatetimeParameterBase): + date_format = ... # type: str + +class IntParameter(Parameter): + def parse(self, s): ... + def next_in_enumeration(self, value): ... + +class FloatParameter(Parameter): + def parse(self, s): ... + +class BoolParameter(Parameter): + def __init__(self, *args, **kwargs) -> None: ... + def parse(self, s): ... + def normalize(self, value): ... + +class DateIntervalParameter(Parameter): + def parse(self, s): ... + +class TimeDeltaParameter(Parameter): + def parse(self, input): ... + def serialize(self, x): ... + +class TaskParameter(Parameter): + def parse(self, input): ... + def serialize(self, cls): ... + +class EnumParameter(Parameter): + def __init__(self, *args, **kwargs) -> None: ... + def parse(self, s): ... + def serialize(self, e): ... + +class FrozenOrderedDict(Mapping): + def __init__(self, *args, **kwargs) -> None: ... + def __getitem__(self, key): ... + def __iter__(self): ... + def __len__(self): ... + def __hash__(self): ... + def get_wrapped(self): ... + +class DictParameter(Parameter): + class DictParamEncoder(JSONEncoder): + def default(self, obj): ... + def normalize(self, value): ... + def parse(self, s): ... + def serialize(self, x): ... + +class ListParameter(Parameter): + def normalize(self, x): ... + def parse(self, x): ... + def serialize(self, x): ... + +class TupleParameter(Parameter): + def parse(self, x): ... + def serialize(self, x): ... + +class NumericalParameter(Parameter): + description = ... # type: str + def __init__(self, left_op: Any = ..., right_op: Any = ..., *args, **kwargs) -> None: ... + def parse(self, s): ... + +class ChoiceParameter(Parameter): + description = ... # type: str + def __init__(self, var_type: Any = ..., *args, **kwargs) -> None: ... + def parse(self, s): ... diff --git a/stubs/luigi/rpc.pyi b/stubs/luigi/rpc.pyi new file mode 100644 index 0000000..b73c49f --- /dev/null +++ b/stubs/luigi/rpc.pyi @@ -0,0 +1,26 @@ +# Stubs for luigi.rpc (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional + +HAS_UNIX_SOCKET = ... # type: bool +HAS_REQUESTS = ... # type: bool +logger = ... # type: Any + +class RPCError(Exception): + sub_exception = ... # type: Any + def __init__(self, message, sub_exception: Optional[Any] = ...) -> None: ... + +class URLLibFetcher: + raises = ... # type: Any + def fetch(self, full_url, body, timeout): ... + +class RequestsFetcher: + raises = ... # type: Any + session = ... # type: Any + def __init__(self, session) -> None: ... + def fetch(self, full_url, body, timeout): ... + +class RemoteScheduler: + def __init__(self, url: str = ..., connect_timeout: Optional[Any] = ...) -> None: ... diff --git a/stubs/luigi/target.pyi b/stubs/luigi/target.pyi new file mode 100644 index 0000000..3903aa4 --- /dev/null +++ b/stubs/luigi/target.pyi @@ -0,0 +1,47 @@ +# Stubs for luigi.target (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any +import io + +logger = ... # type: Any + +class Target: + def exists(self) -> bool: ... + +class FileSystemException(Exception): ... +class FileAlreadyExists(FileSystemException): ... +class MissingParentDirectory(FileSystemException): ... +class NotADirectory(FileSystemException): ... + +class FileSystem: + def exists(self, path): ... + def remove(self, path, recursive: bool = ..., skip_trash: bool = ...): ... + def mkdir(self, path, parents: bool = ..., raise_if_exists: bool = ...): ... + def isdir(self, path): ... + def listdir(self, path): ... + def move(self, path, dest): ... + def rename_dont_move(self, path, dest): ... + def rename(self, *args, **kwargs): ... + def copy(self, path, dest): ... + +class FileSystemTarget(Target): + path = ... # type: Any + def __init__(self, path) -> None: ... + def fs(self): ... + def open(self, mode): ... + def exists(self): ... + def remove(self): ... + def temporary_path(self): ... + +class AtomicLocalFile(io.BufferedWriter): + path = ... # type: Any + def __init__(self, path) -> None: ... + def close(self): ... + def generate_tmp_path(self, path): ... + def move_to_final_destination(self): ... + def __del__(self): ... + @property + def tmp_path(self): ... + def __exit__(self, exc_type, exc, traceback): ... diff --git a/stubs/luigi/task.pyi b/stubs/luigi/task.pyi new file mode 100644 index 0000000..66e892c --- /dev/null +++ b/stubs/luigi/task.pyi @@ -0,0 +1,107 @@ +# Stubs for luigi.task (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Dict, Generic, Iterable, List, Optional, TypeVar, Type, Union + +from .target import Target + +Parameter = ... # type: Any +logger = ... # type: Any +TASK_ID_INCLUDE_PARAMS = ... # type: int +TASK_ID_TRUNCATE_PARAMS = ... # type: int +TASK_ID_TRUNCATE_HASH = ... # type: int +TASK_ID_INVALID_CHAR_REGEX = ... # type: Any + +def namespace(namespace: Optional[Any] = ..., scope: str = ...): ... +def auto_namespace(scope: str = ...): ... +def task_id_str(task_family, params): ... + +class BulkCompleteNotImplementedError(NotImplementedError): ... + +_R = TypeVar('_R') +_I = TypeVar('_I') +_O = TypeVar('_O') +_T = TypeVar('_T') + +class Task(Generic[_R, _I, _O]): + priority = ... # type: int + disabled = ... # type: bool + resources = ... # type: Any + worker_timeout = ... # type: Any + max_batch_size = ... # type: Any + @property + def batchable(self): ... + @property + def retry_count(self): ... + @property + def disable_hard_timeout(self): ... + @property + def disable_window_seconds(self): ... + @property + def owner_email(self): ... + @property + def use_cmdline_section(self): ... + @classmethod + def event_handler(cls, event): ... + def trigger_event(self, event, *args, **kwargs): ... + @property + def task_module(self): ... + task_namespace = ... # type: Any + @classmethod + def get_task_namespace(cls): ... + @property + def task_family(self): ... + @classmethod + def get_task_family(cls): ... + @classmethod + def get_params(cls): ... + @classmethod + def batch_param_names(cls): ... + @classmethod + def get_param_names(cls, include_significant: bool = ...) -> List[str]: ... + @classmethod + def get_param_values(cls, params, args, kwargs): ... + param_args = ... # type: Any + param_kwargs = ... # type: Any + task_id = ... # type: str + set_tracking_url = ... # type: Any + set_status_message = ... # type: Any + def __init__(self, *args, **kwargs) -> None: ... + def initialized(self): ... + @classmethod + def from_str_params(cls: Type[_T], params_str: Dict[str, str]) -> _T: ... + def to_str_params(self, only_significant: bool = ...): ... + def clone(self, cls: Optional[Any] = ..., **kwargs): ... + def __hash__(self): ... + def __eq__(self, other): ... + def complete(self) -> bool: ... + @classmethod + def bulk_complete(cls, parameter_tuples): ... + def output(self) -> _O: ... + def requires(self) -> _R: ... + def process_resources(self): ... + def input(self) -> _I: ... + def deps(self): ... + def run(self) -> None: ... + def on_failure(self, exception): ... + def on_success(self): ... + def no_unpicklable_properties(self): ... + +class MixinNaiveBulkComplete: + @classmethod + def bulk_complete(cls, parameter_tuples): ... + +class ExternalTask(Task[_R, _I, _O]): + run = ... # type: Any + +def externalize(taskclass_or_taskobject): ... + +class WrapperTask(Task[_R, _I, _O]): + def complete(self): ... + +class Config(Task[_R, _I, _O]): ... + +def getpaths(struct): ... +def flatten(struct: Union[Dict[Any, _O], Iterable[_O], _O]) -> List[_O]: ... +def flatten_output(task): ... diff --git a/stubs/luigi/tools/__init__.pyi b/stubs/luigi/tools/__init__.pyi new file mode 100644 index 0000000..d516037 --- /dev/null +++ b/stubs/luigi/tools/__init__.pyi @@ -0,0 +1,4 @@ +# Stubs for luigi.tools (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + diff --git a/stubs/luigi/tools/range.pyi b/stubs/luigi/tools/range.pyi new file mode 100644 index 0000000..ad0de09 --- /dev/null +++ b/stubs/luigi/tools/range.pyi @@ -0,0 +1,86 @@ +# Stubs for luigi.tools.range (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any +import luigi + +logger = ... # type: Any + +class RangeEvent(luigi.Event): + COMPLETE_COUNT = ... # type: str + COMPLETE_FRACTION = ... # type: str + DELAY = ... # type: str + +class RangeBase(luigi.WrapperTask): + of = ... # type: Any + of_params = ... # type: Any + start = ... # type: Any + stop = ... # type: Any + reverse = ... # type: Any + task_limit = ... # type: Any + now = ... # type: Any + param_name = ... # type: Any + @property + def of_cls(self): ... + def datetime_to_parameter(self, dt): ... + def parameter_to_datetime(self, p): ... + def datetime_to_parameters(self, dt): ... + def parameters_to_datetime(self, p): ... + def moving_start(self, now): ... + def moving_stop(self, now): ... + def finite_datetimes(self, finite_start, finite_stop): ... + def requires(self): ... + def missing_datetimes(self, finite_datetimes): ... + +class RangeDailyBase(RangeBase): + start = ... # type: Any + stop = ... # type: Any + days_back = ... # type: Any + days_forward = ... # type: Any + def datetime_to_parameter(self, dt): ... + def parameter_to_datetime(self, p): ... + def datetime_to_parameters(self, dt): ... + def parameters_to_datetime(self, p): ... + def moving_start(self, now): ... + def moving_stop(self, now): ... + def finite_datetimes(self, finite_start, finite_stop): ... + +class RangeHourlyBase(RangeBase): + start = ... # type: Any + stop = ... # type: Any + hours_back = ... # type: Any + hours_forward = ... # type: Any + def datetime_to_parameter(self, dt): ... + def parameter_to_datetime(self, p): ... + def datetime_to_parameters(self, dt): ... + def parameters_to_datetime(self, p): ... + def moving_start(self, now): ... + def moving_stop(self, now): ... + def finite_datetimes(self, finite_start, finite_stop): ... + +class RangeByMinutesBase(RangeBase): + start = ... # type: Any + stop = ... # type: Any + minutes_back = ... # type: Any + minutes_forward = ... # type: Any + minutes_interval = ... # type: Any + def datetime_to_parameter(self, dt): ... + def parameter_to_datetime(self, p): ... + def datetime_to_parameters(self, dt): ... + def parameters_to_datetime(self, p): ... + def moving_start(self, now): ... + def moving_stop(self, now): ... + def finite_datetimes(self, finite_start, finite_stop): ... + +def most_common(items): ... +def infer_bulk_complete_from_fs(datetimes, datetime_to_task, datetime_to_re): ... + +class RangeDaily(RangeDailyBase): + def missing_datetimes(self, finite_datetimes): ... + +class RangeHourly(RangeHourlyBase): + def missing_datetimes(self, finite_datetimes): ... + +class RangeByMinutes(RangeByMinutesBase): + def missing_datetimes(self, finite_datetimes): ... diff --git a/stubs/sqlalchemy/__init__.pyi b/stubs/sqlalchemy/__init__.pyi new file mode 100644 index 0000000..8d58e25 --- /dev/null +++ b/stubs/sqlalchemy/__init__.pyi @@ -0,0 +1,124 @@ +# Stubs for sqlalchemy (Python 2) + +from .sql import ( + alias, + and_, + asc, + between, + bindparam, + case, + cast, + collate, + column, + delete, + desc, + distinct, + except_, + except_all, + exists, + extract, + false, + func, + funcfilter, + insert, + intersect, + intersect_all, + join, + literal, + literal_column, + modifier, + not_, + null, + or_, + outerjoin, + outparam, + over, + select, + subquery, + table, + text, + true, + tuple_, + type_coerce, + union, + union_all, + update, +) + +from .types import ( + BIGINT, + BINARY, + BLOB, + BOOLEAN, + BigInteger, + Binary, + Boolean, + CHAR, + CLOB, + DATE, + DATETIME, + DECIMAL, + Date, + DateTime, + Enum, + FLOAT, + Float, + INT, + INTEGER, + Integer, + Interval, + LargeBinary, + NCHAR, + NVARCHAR, + NUMERIC, + Numeric, + PickleType, + REAL, + SMALLINT, + SmallInteger, + String, + TEXT, + TIME, + TIMESTAMP, + Text, + Time, + TypeDecorator, + Unicode, + UnicodeText, + VARBINARY, + VARCHAR, +) + +from .schema import ( + CheckConstraint, + Column, + ColumnDefault, + Constraint, + DefaultClause, + FetchedValue, + ForeignKey, + ForeignKeyConstraint, + Index, + MetaData, + PassiveDefault, + PrimaryKeyConstraint, + Sequence, + Table, + ThreadLocalMetaData, + UniqueConstraint, + DDL, +) + +from . import sql as sql +from . import schema as schema +from . import types as types +from . import exc as exc +from . import dialects as dialects +from . import pool as pool +# This should re-export orm but orm is totally broken right now +# from . import orm as orm + +from .inspection import inspect +from .engine import create_engine, engine_from_config + +__version__ = ... # type: int diff --git a/stubs/sqlalchemy/engine/__init__.pyi b/stubs/sqlalchemy/engine/__init__.pyi new file mode 100644 index 0000000..396cac5 --- /dev/null +++ b/stubs/sqlalchemy/engine/__init__.pyi @@ -0,0 +1,11 @@ +# Stubs for sqlalchemy.engine (Python 2) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from .base import Connection as Connection +from .base import Engine as Engine +from .base import Transaction as Transaction +from .result import RowProxy as RowProxy + +def create_engine(*args, **kwargs): ... +def engine_from_config(configuration, prefix=..., **kwargs): ... diff --git a/stubs/sqlalchemy/engine/base.pyi b/stubs/sqlalchemy/engine/base.pyi new file mode 100644 index 0000000..4f5ff43 --- /dev/null +++ b/stubs/sqlalchemy/engine/base.pyi @@ -0,0 +1,39 @@ +from typing import ( + Any, Generic, List, Optional, Tuple, Type, TypeVar +) +from types import TracebackType + +from .result import ResultProxy + +_T = TypeVar('_T') + +# new in 3.6 +class ContextManager(Generic[_T]): + def __enter__(self) -> _T: ... + def __exit__(self, exc_type: Optional[Type[BaseException]], + exc_val: Optional[Exception], + exc_tb: Optional[TracebackType]) -> bool: ... + + +# Dummy until I figure out something better. +class Connectable: + pass + +class Connection: + def begin(self) -> 'Transaction': ... + def execute(self, object: object, *multiparams: Any, **params: Any) -> ResultProxy: ... + def scalar(self, object: object, *multiparams: Any, **params: Any) -> Any: ... + +class Engine(object): + def execute(self, object: object, *multiparams: Any, **params: Any) -> ResultProxy: ... + def connect(self) -> Connection: ... + +class RowProxy: + def items(self) -> List[Tuple[Any, Any]]: ... + def keys(self) -> List[Any]: ... + def values(self) -> List[Any]: ... + def __getitem__(self, key: str): ... + +class Transaction(ContextManager): + def commit(self): ... + def rollback(self): ... diff --git a/stubs/sqlalchemy/engine/result.pyi b/stubs/sqlalchemy/engine/result.pyi new file mode 100644 index 0000000..a44553b --- /dev/null +++ b/stubs/sqlalchemy/engine/result.pyi @@ -0,0 +1,86 @@ +# Stubs for sqlalchemy.engine.result (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Iterable, List, Optional +from ..sql import expression as expression, sqltypes as sqltypes +# from ..sql import util as sql_util +# from sqlalchemy.cresultproxy import BaseRowProxy + +def rowproxy_reconstructor(cls, state): ... + +class BaseRowProxy(Iterable): + def __init__(self, parent, row, processors, keymap) -> None: ... + def __reduce__(self): ... + def values(self): ... + def __iter__(self): ... + def __len__(self): ... + def __getitem__(self, key): ... + def __getattr__(self, name): ... + +class RowProxy(BaseRowProxy): + def __contains__(self, key): ... + __hash__ = ... # type: Any + def __lt__(self, other): ... + def __le__(self, other): ... + def __ge__(self, other): ... + def __gt__(self, other): ... + def __eq__(self, other): ... + def __ne__(self, other): ... + def has_key(self, key): ... + def items(self): ... + def keys(self) -> List[str]: ... + def iterkeys(self): ... + def itervalues(self): ... + +class ResultMetaData: + case_sensitive = ... # type: Any + matched_on_name = ... # type: bool + def __init__(self, parent, cursor_description) -> None: ... + +class ResultProxy: + out_parameters = ... # type: Any + closed = ... # type: bool + context = ... # type: Any + dialect = ... # type: Any + cursor = ... # type: Any + connection = ... # type: Any + def __init__(self, context) -> None: ... + def keys(self) -> List[str]: ... + rowcount = ... # type: int + @property + def lastrowid(self): ... + @property + def returns_rows(self): ... + @property + def is_insert(self): ... + def close(self): ... + def __iter__(self): ... + def inserted_primary_key(self): ... + def last_updated_params(self): ... + def last_inserted_params(self): ... + @property + def returned_defaults(self): ... + def lastrow_has_defaults(self): ... + def postfetch_cols(self): ... + def prefetch_cols(self): ... + def supports_sane_rowcount(self): ... + def supports_sane_multi_rowcount(self): ... + def process_rows(self, rows): ... + def fetchall(self) -> List[RowProxy]: ... + def fetchmany(self, size: Optional[Any] = ...): ... + def fetchone(self) -> RowProxy: ... + def first(self): ... + def scalar(self): ... + +class BufferedRowResultProxy(ResultProxy): + size_growth = ... # type: Any + +class FullyBufferedResultProxy(ResultProxy): ... + +class BufferedColumnRow(RowProxy): + def __init__(self, parent, row, processors, keymap) -> None: ... + +class BufferedColumnResultProxy(ResultProxy): + def fetchall(self) -> List[RowProxy]: ... + def fetchmany(self, size: Optional[Any] = ...) -> List[RowProxy]: ... diff --git a/stubs/sqlalchemy/engine/url.pyi b/stubs/sqlalchemy/engine/url.pyi new file mode 100644 index 0000000..7880ccc --- /dev/null +++ b/stubs/sqlalchemy/engine/url.pyi @@ -0,0 +1,27 @@ +# Stubs for sqlalchemy.engine.url (Python 2) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, AnyStr +from .. import dialects + +# registry = dialects.registry + +class URL: + drivername = ... # type: Any + username = ... # type: Any + password = ... # type: Any + host = ... # type: Any + port = ... # type: Any + database = ... # type: Any + query = ... # type: Any + def __init__(self, drivername, username=..., password=..., host=..., port=..., database=..., query=...) -> None: ... + def __to_string__(self, hide_password=...): ... + def __hash__(self): ... + def __eq__(self, other): ... + def get_backend_name(self): ... + def get_driver_name(self): ... + def get_dialect(self): ... + def translate_connect_args(self, names=..., **kw): ... + +def make_url(name_or_url: AnyStr) -> URL: ... diff --git a/stubs/sqlalchemy/exc.pyi b/stubs/sqlalchemy/exc.pyi new file mode 100644 index 0000000..ef3c7af --- /dev/null +++ b/stubs/sqlalchemy/exc.pyi @@ -0,0 +1,27 @@ +# Stubs for sqlalchemy.exc (Python 3.5) +# +# NOTE: This dynamically typed is an excerpt from a +# stub automatically generated by stubgen. + +from typing import Any, Optional + +class SQLAlchemyError(Exception): ... + +class StatementError(SQLAlchemyError): + statement = ... # type: Any + params = ... # type: Any + orig = ... # type: Any + detail = ... # type: Any + def __init__(self, message, statement, params, orig) -> None: ... + def add_detail(self, msg): ... + def __reduce__(self): ... + def __unicode__(self): ... + +class DBAPIError(StatementError): + @classmethod + def instance(cls, statement, params, orig, dbapi_base_err, connection_invalidated: bool = ..., dialect: Optional[Any] = ...): ... + def __reduce__(self): ... + connection_invalidated = ... # type: Any + def __init__(self, statement, params, orig, connection_invalidated: bool = ...) -> None: ... + +class DatabaseError(DBAPIError): ... diff --git a/stubs/sqlalchemy/sql/__init__.pyi b/stubs/sqlalchemy/sql/__init__.pyi new file mode 100644 index 0000000..e69de29 diff --git a/stubs/sqlalchemy/sql/elements.pyi b/stubs/sqlalchemy/sql/elements.pyi new file mode 100644 index 0000000..eabe5b3 --- /dev/null +++ b/stubs/sqlalchemy/sql/elements.pyi @@ -0,0 +1,25 @@ +from typing import Any, Optional + +class Visitable(object): + pass + +class ClauseElement(Visitable): + __visit_name__ = ... # type: str + supports_execution = ... # type: bool + bind = ... # type: Any + is_selectable = ... # type: bool + is_clause_element = ... # type: bool + description = ... # type: Any + def unique_params(self, *optionaldict, **kwargs): ... + def params(self, *optionaldict, **kwargs): ... + def compare(self, other, **kw): ... + def get_children(self, **kwargs): ... + def self_group(self, against: Optional[Any] = ...): ... + # @util.dependencies("sqlalchemy.engine.default") + # def compile(self, default, bind: Optional[Any] = ..., dialect: Optional[Any] = ..., **kw): ... + def compile(self, bind: Optional[Any] = ..., dialect: Optional[Any] = ..., **kw): ... + def __and__(self, other): ... + def __or__(self, other): ... + def __invert__(self): ... + def __bool__(self): ... + __nonzero__ = ... # type: Any diff --git a/stubs/sqlalchemy/sql/expression.pyi b/stubs/sqlalchemy/sql/expression.pyi new file mode 100644 index 0000000..f361bd2 --- /dev/null +++ b/stubs/sqlalchemy/sql/expression.pyi @@ -0,0 +1,7 @@ +from .elements import ClauseElement + +class Select(ClauseElement): + @property + def columns(self): ... + c = ... # type: Any + def where(self, whereclause): ... From 29d59e858a9b3d126e31ef148e5a9a22e1654fa5 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 25 Jan 2018 16:37:53 -0600 Subject: [PATCH 315/507] import heron_tasks luigi support https://github.com/kumc-bmi/heron/tree/para_etl_3985/heron_tasks 21fa46c on Dec 13, 2017 --- Oracle/PCORNetInit.sql | 84 ++++++++++++++++++++++++++++++++++++ Oracle/PCORNetLoader_ora.sql | 4 +- Oracle/med_admin.sql | 64 +++++++++++++++++++++++++++ i2p_tasks.py | 24 ++++++++++- script_lib.py | 4 ++ 5 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 Oracle/med_admin.sql diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index 6149995..d00b9a7 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -1007,4 +1007,88 @@ CREATE TABLE harvest( REFRESH_DEATH_CAUSE_DATE date NULL ) / + +-------------------------------------------------------------------------------- +-- OBS_CLIN +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE obs_clin'); +END; +/ +CREATE TABLE obs_clin( + OBSCLINID varchar(50) NOT NULL, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, + OBSCLIN_PROVIDERID varchar(50) NULL, + OBSCLIN_DATE date NULL, + OBSCLIN_TIME varchar(5) NULL, + OBSCLIN_TYPE varchar(2) NULL, + OBSCLIN_CODE varchar(50) NULL, + OBSCLIN_RESULT_QUAL varchar(50) NULL, + OBSCLIN_RESULT_TEXT varchar(50) NULL, + OBSCLIN_RESULT_SNOMED varchar(50), + OBSCLIN_RESULT_NUM NUMBER(18, 0) NULL, -- (8,0) + OBSCLIN_RESULT_MODIFIER varchar(2) NULL, + OBSCLIN_RESULT_UNIT varchar(50) NULL, + RAW_OBSCLIN_NAME varchar(50) NULL, + RAW_OBSCLIN_CODE varchar(50) NULL, + RAW_OBSCLIN_TYPE varchar(50) NULL, + RAW_OBSCLIN_RESULT varchar(50) NULL, + RAW_OBSCLIN_MODIFIER varchar(50) NULL, + RAW_OBSCLIN_UNIT varchar(50) NULL +) +/ + +-------------------------------------------------------------------------------- +-- OBS_GEN +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE obs_gen'); +END; +/ +CREATE TABLE obs_gen( + OBSGENID varchar(50) NOT NULL, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, + OBSGEN_PROVIDERID varchar(50) NULL, + OBSGEN_DATE date NULL, + OBSGEN_TIME varchar(5) NULL, + OBSGEN_TYPE varchar(30) NULL, + OBSGEN_CODE varchar(50) NULL, + OBSGEN_RESULT_QUAL varchar(50) NULL, + OBSGEN_RESULT_TEXT varchar(50) NULL, + OBSGEN_RESULT_NUM NUMBER(18, 0) NULL, -- (8,0) + OBSGEN_RESULT_MODIFIER varchar(2) NULL, + OBSGEN_RESULT_UNIT varchar(50) NULL, + OBSGEN_TABLE_MODIFIED varchar(3) NULL, + OBSGEN_ID_MODIFIED varchar(50) NULL, + RAW_OBSGEN_NAME varchar(50) NULL, + RAW_OBSGEN_CODE varchar(50) NULL, + RAW_OBSGEN_TYPE varchar(50) NULL, + RAW_OBSGEN_RESULT varchar(50) NULL, + RAW_OBSGEN_UNIT varchar(50) NULL +) +/ + +-------------------------------------------------------------------------------- +-- PROVIDER +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE provider'); +END; +/ +CREATE TABLE provider( + PROVIDERID varchar(50) NOT NULL, + PROVIDER_SEX varchar(2) NULL, + PROVIDER_SPECIALTY_PRIMARY varchar(50) NULL, + PROVIDER_NPI NUMBER(18, 0) NULL, -- (8,0) + PROVIDER_NPI_FLAG varchar(1) NULL, + RAW_PROVIDER_SPECIALTY_PRIMARY varchar(50) NULL +) +/ + + SELECT 1 FROM HARVEST diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index a2ecd11..8a28e78 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1288,4 +1288,6 @@ begin PCORNetPostProc; end PCORNetLoader; -/ \ No newline at end of file +/ + +SELECT 1 FROM HARVEST \ No newline at end of file diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql new file mode 100644 index 0000000..f54eb02 --- /dev/null +++ b/Oracle/med_admin.sql @@ -0,0 +1,64 @@ +-------------------------------------------------------------------------------- +-- MED_ADMIN +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE med_admin'); +END; +/ +CREATE TABLE med_admin( + MEDADMINID varchar(50) primary key, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, + PRESCRIBINGID varchar(50) NULL, + MEDADMIN_PROVIDERID varchar(50) NULL, + MEDADMIN_START_DATE date NOT NULL, + MEDADMIN_START_TIME varchar(5) NULL, + MEDADMIN_STOP_DATE date NULL, + MEDADMIN_STOP_TIME varchar(5) NULL, + MEDADMIN_TYPE varchar(2) NULL, + MEDADMIN_CODE varchar(50) NULL, + MEDADMIN_DOSE_ADMIN NUMBER(18, 0) NULL, -- (8,0) + MEDADMIN_DOSE_ADMIN_UNIT varchar(50) NULL, + MEDADMIN_ROUTE varchar(50) NULL, + MEDADMIN_SOURCE varchar(2) NULL, + RAW_MEDADMIN_MED_NAME varchar(50) NULL, + RAW_MEDADMIN_CODE varchar(50) NULL, + RAW_MEDADMIN_DOSE_ADMIN varchar(50) NULL, + RAW_MEDADMIN_DOSE_ADMIN_UNIT varchar(50) NULL, + RAW_MEDADMIN_ROUTE varchar(50) NULL +) +/ +BEGIN +PMN_DROPSQL('DROP sequence med_admin_seq'); +END; +/ +create sequence med_admin_seq +/ +create or replace trigger med_admin_trg +before insert on med_admin +for each row +begin + select med_admin_seq.nextval into :new.MEDADMINID from dual; +end; +/ +create or replace procedure PCORNetMedAdmin as +begin + +PMN_DROPSQL('drop index med_admin_idx'); + +execute immediate 'truncate table med_admin'; + +insert into med_admin(patid, encounterid, medadmin_providerid, medadmin_start_date, medadmin_start_time, medadmin_stop_date, +medadmin_stop_time, medadmin_type, medadmin_code, medadmin_source) +select patient_num, encounter_num, provider_id, start_date, to_char(start_date, 'HH24:MI'), end_date, to_char(end_date, 'HH24:MI'), +'RX' as medadmin_type, concept_cd, 'OD' as medadmin_source +from pcornet_cdm.observation_fact_meds; + +execute immediate 'create index med_admin_idx on prescribing (PATID, ENCOUNTERID)'; +GATHER_TABLE_STATS('MED_ADMIN'); + +end PCORNetMedAdmin; +/ + +SELECT 1 FROM HARVEST \ No newline at end of file diff --git a/i2p_tasks.py b/i2p_tasks.py index 553bece..805d696 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -1,6 +1,8 @@ from etl_tasks import SqlScriptTask +import luigi from script_lib import Script from sql_syntax import Environment +from typing import List class PCORNetInit(SqlScriptTask): @@ -8,4 +10,24 @@ class PCORNetInit(SqlScriptTask): @property def variables(self) -> Environment: - return dict(datamart_id='todo', datamart_name='todo2', i2b2_data_schema='todo3') + return dict(datamart_id='todo1', datamart_name='todo2', i2b2_data_schema='BLUEHERONDATA', min_pat_list_date_dd_mon_rrrr='20-JAN-18', min_visit_date_dd_mon_rrrr='20-JAN-18', + i2b2_meta_schema='BLUEHERONMETADATA') + +class PCORNetLoader_ora(SqlScriptTask): + script = Script.PCORNetLoader_ora + + @property + def variables(self) -> Environment: + return dict(i2b2_meta_schema='BLUEHERONMETADATA', network_id='todo1', network_name='todo2', enrollment_months_back='2') + +class PCORNetMed_Admin(SqlScriptTask): + script = Script.med_admin + + @property + def variables(self) -> Environment: + return dict(i2b2_meta_schema='BLUEHERONMETADATA', network_id='todo1', network_name='todo2', enrollment_months_back='2') + + def requires(self) -> List[luigi.Task]: + '''Wrap each of `self.script.deps()` in a SqlScriptTask. + ''' + return [PCORNetInit(), PCORNetLoader_ora()] \ No newline at end of file diff --git a/script_lib.py b/script_lib.py index c13e22c..a066781 100644 --- a/script_lib.py +++ b/script_lib.py @@ -266,18 +266,22 @@ class Script(ScriptMixin, enum.Enum): [ # Keep sorted PCORNetInit, + PCORNetLoader_ora, epic_facts_load, epic_flowsheets_transform, etl_tests_init, + med_admin, migrate_fact_upload, ] = [ pkg.resource_string(__name__, 'Oracle/' + fname).decode('utf-8') for fname in [ 'PCORNetInit.sql', + 'PCORNetLoader_ora.sql', 'epic_facts_load.sql', 'epic_flowsheets_transform.sql', 'etl_tests_init.sql', + 'med_admin.sql', 'migrate_fact_upload.sql', ] ] From 733d6b246568887661b191d911702b6d5cf753ad Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 29 Jan 2018 10:36:45 -0600 Subject: [PATCH 316/507] Adde Oracle loader and Med_Admin table creation to the Luigi task list. https://github.com/kumc-bmi/heron/tree/para_etl_3985/heron_tasks 21fa46c on Dec 13, 2017 --- Oracle/PCORNetInit.sql | 14 +++++++------- Oracle/PCORNetLoader_ora.sql | 2 +- Oracle/med_admin.sql | 25 ++++++++++++++++++------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index d00b9a7..b38a65e 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -600,9 +600,9 @@ CREATE TABLE LOCATION ( / -- TODO: This seems to belong in h2p-mapping -alter table "&&i2b2_meta_schema".pcornet_lab add ( - pcori_specimen_source varchar2(1000) -- arbitrary - ); +--alter table "&&i2b2_meta_schema".pcornet_lab add ( +-- pcori_specimen_source varchar2(1000) -- arbitrary +-- ); -------------------------------------------------------------------------------- @@ -893,9 +893,9 @@ CREATE TABLE AMOUNT ( / -- TODO: This seems to belong in h2p-mapping -alter table "&&i2b2_meta_schema".pcornet_med add ( - pcori_ndc varchar2(1000) -- arbitrary - ); +--alter table "&&i2b2_meta_schema".pcornet_med add ( +-- pcori_ndc varchar2(1000) -- arbitrary +-- ); -------------------------------------------------------------------------------- @@ -1091,4 +1091,4 @@ CREATE TABLE provider( / -SELECT 1 FROM HARVEST +SELECT * FROM dual where rownum < 2 diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 8a28e78..1fd6721 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1290,4 +1290,4 @@ begin end PCORNetLoader; / -SELECT 1 FROM HARVEST \ No newline at end of file +SELECT * FROM dual where rownum < 2 \ No newline at end of file diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index f54eb02..a586513 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -18,7 +18,7 @@ CREATE TABLE med_admin( MEDADMIN_STOP_TIME varchar(5) NULL, MEDADMIN_TYPE varchar(2) NULL, MEDADMIN_CODE varchar(50) NULL, - MEDADMIN_DOSE_ADMIN NUMBER(18, 0) NULL, -- (8,0) + MEDADMIN_DOSE_ADMIN NUMBER(18, 2) NULL, -- (8,0) MEDADMIN_DOSE_ADMIN_UNIT varchar(50) NULL, MEDADMIN_ROUTE varchar(50) NULL, MEDADMIN_SOURCE varchar(2) NULL, @@ -50,13 +50,24 @@ PMN_DROPSQL('drop index med_admin_idx'); execute immediate 'truncate table med_admin'; insert into med_admin(patid, encounterid, medadmin_providerid, medadmin_start_date, medadmin_start_time, medadmin_stop_date, -medadmin_stop_time, medadmin_type, medadmin_code, medadmin_source) -select patient_num, encounter_num, provider_id, start_date, to_char(start_date, 'HH24:MI'), end_date, to_char(end_date, 'HH24:MI'), -'RX' as medadmin_type, concept_cd, 'OD' as medadmin_source -from pcornet_cdm.observation_fact_meds; +medadmin_stop_time, medadmin_type, medadmin_code, medadmin_dose_admin, medadmin_dose_admin_unit, medadmin_source, raw_medadmin_code, +raw_medadmin_dose_admin, raw_medadmin_dose_admin_unit) -execute immediate 'create index med_admin_idx on prescribing (PATID, ENCOUNTERID)'; -GATHER_TABLE_STATS('MED_ADMIN'); +with med_start as ( + select patient_num, encounter_num, provider_id, start_date, end_date, concept_cd, modifier_cd, instance_num + from pcornet_cdm.observation_fact_meds + where (modifier_cd like '%MAR%New Bag%' or (modifier_cd like '%MAR%Given%' and modifier_cd not like '%Not Given%')) +) +select med_start.patient_num, med_start.encounter_num, med_start.provider_id, med_start.start_date, to_char(med_start.start_date, 'HH24:MI'), med_start.end_date, to_char(med_start.end_date, 'HH24:MI'), +'RX', med_start.concept_cd, med_dose.nval_num, med_dose.units_cd, 'OD', med_start.concept_cd, med_dose.nval_num, med_dose.units_cd +from med_start +left join pcornet_cdm.observation_fact_meds med_dose +on med_dose.instance_num = med_start.instance_num +and med_dose.modifier_cd like '%Dose%' +; + +execute immediate 'create index med_admin_idx on med_admin (PATID, ENCOUNTERID)'; +--GATHER_TABLE_STATS('MED_ADMIN'); end PCORNetMedAdmin; / From 74b21a87f95656b8f7afa1959956c7ef80fd6bfd Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 31 Jan 2018 11:46:52 -0600 Subject: [PATCH 317/507] Added additional modifier codes and pcornet_med data to med_admin insert statement --- Oracle/med_admin.sql | 58 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index a586513..7831914 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -50,20 +50,59 @@ PMN_DROPSQL('drop index med_admin_idx'); execute immediate 'truncate table med_admin'; insert into med_admin(patid, encounterid, medadmin_providerid, medadmin_start_date, medadmin_start_time, medadmin_stop_date, -medadmin_stop_time, medadmin_type, medadmin_code, medadmin_dose_admin, medadmin_dose_admin_unit, medadmin_source, raw_medadmin_code, -raw_medadmin_dose_admin, raw_medadmin_dose_admin_unit) +medadmin_stop_time, medadmin_type, medadmin_code, medadmin_dose_admin, medadmin_dose_admin_unit, medadmin_source, raw_medadmin_med_name, +raw_medadmin_code, raw_medadmin_dose_admin, raw_medadmin_dose_admin_unit, raw_medadmin_route) with med_start as ( select patient_num, encounter_num, provider_id, start_date, end_date, concept_cd, modifier_cd, instance_num from pcornet_cdm.observation_fact_meds - where (modifier_cd like '%MAR%New Bag%' or (modifier_cd like '%MAR%Given%' and modifier_cd not like '%Not Given%')) + where modifier_cd = 'MedObs|MAR:New Bag' + or modifier_cd = 'MedObs|MAR:Downtime Given - New Bag' + or modifier_cd = 'MedObs|MAR:Given - Without Order' + or modifier_cd = 'MedObs|MAR:Downtime Given' + or modifier_cd = 'MedObs|MAR:Given - Without Order' + or modifier_cd = 'MedObs|MAR:Given by another Provider' + or modifier_cd = 'MedObs|MAR:Given by Patient/Family' + or modifier_cd = 'MedObs|MAR:Given' + or modifier_cd = 'MedObs|MAR:Clinic Administered - Patient Supplied' + or modifier_cd = 'MedObs|MAR:Bolus' + or modifier_cd = 'MedObs|MAR:Per Protocol' + or modifier_cd = 'MedObs|MAR:Infusion Home with Patient' + or modifier_cd = 'MedObs|MAR:ED-Infusing Upon Admit' + or modifier_cd = 'MedObs|MAR:Push' + or modifier_cd = 'MedObs|MAR:Restarted' + or modifier_cd = 'MedObs|MAR:PCA Check/Change' + or modifier_cd = 'MedObs|MAR:NPO' + or modifier_cd = 'MedObs|MAR:See OR/Proc Flowsheet' + or modifier_cd = 'MedObs|MAR:Patch Applied' + or modifier_cd = 'MedObs|MAR:Bolus from Syringe' + or modifier_cd = 'MedObs|MAR:Bolus.' ) -select med_start.patient_num, med_start.encounter_num, med_start.provider_id, med_start.start_date, to_char(med_start.start_date, 'HH24:MI'), med_start.end_date, to_char(med_start.end_date, 'HH24:MI'), -'RX', med_start.concept_cd, med_dose.nval_num, med_dose.units_cd, 'OD', med_start.concept_cd, med_dose.nval_num, med_dose.units_cd +select med_start.patient_num, med_start.encounter_num, med_start.provider_id, med_start.start_date, to_char(med_start.start_date, 'HH24:MI'), med_start.end_date, +to_char(med_start.end_date, 'HH24:MI'), 'RX', med_p.pcori_basecode, med_dose.nval_num, med_dose.units_cd, 'OD', med_p.c_name, med_start.concept_cd, med_dose.nval_num, +med_dose.units_cd, med_start.modifier_cd from med_start left join pcornet_cdm.observation_fact_meds med_dose on med_dose.instance_num = med_start.instance_num -and med_dose.modifier_cd like '%Dose%' +and med_dose.start_date = med_start.start_date +and (med_dose.modifier_cd = 'MedObs:MAR_Dose|puff' +or med_dose.modifier_cd = 'MedObs:MAR_Dose|cap' +or med_dose.modifier_cd = 'MedObs:MAR_Dose|drop' +or med_dose.modifier_cd = 'MedObs:MAR_Dose|meq' +or med_dose.modifier_cd = 'MedObs:MAR_Dose|tab' +or med_dose.modifier_cd = 'MedObs:MAR_Dose|units' +or med_dose.modifier_cd = 'MedObs:MAR_Dose|l' +or med_dose.modifier_cd = 'MedObs:MAR_Dose|mg' +or med_dose.modifier_cd = 'MedObs:Dose|puff' +or med_dose.modifier_cd = 'MedObs:Dose|drop' +or med_dose.modifier_cd = 'MedObs:Dose|cap' +or med_dose.modifier_cd = 'MedObs:Dose|meq' +or med_dose.modifier_cd = 'MedObs:Dose|units' +or med_dose.modifier_cd = 'MedObs:Dose|l' +or med_dose.modifier_cd = 'MedObs:Dose|tab' +or med_dose.modifier_cd = 'MedObs:Dose|mg') +left join BLUEHERONMETADATA.pcornet_med med_p +on med_p.c_basecode = med_start.concept_cd ; execute immediate 'create index med_admin_idx on med_admin (PATID, ENCOUNTERID)'; @@ -71,5 +110,8 @@ execute immediate 'create index med_admin_idx on med_admin (PATID, ENCOUNTERID)' end PCORNetMedAdmin; / - -SELECT 1 FROM HARVEST \ No newline at end of file +BEGIN +PCORNetMedAdmin(); +END; +/ +SELECT count(*) from med_admin \ No newline at end of file From f705b8265fd45478757ee2a5932b40e47b54cb3f Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 14 Feb 2018 14:04:41 -0600 Subject: [PATCH 318/507] Added additional modifier codes and pcornet_med data to med_admin insert statement --- logging.cfg | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/logging.cfg b/logging.cfg index efc350e..798c566 100644 --- a/logging.cfg +++ b/logging.cfg @@ -6,18 +6,6 @@ # https://pypi.python.org/pypi/python-json-logger # https://stedolan.github.io/jq/manual/v1.5/ # -# Logging to JSON lets us do fun stuff such as: -# -# $ < log/grouse-detail.json jq --compact-output -C \ -# 'select(.args|objects|.event == "inserted chunk") | -# [.asctime, .process, .args.filename, .args.lineno, .args.into, -# {elapsed: (.elapsed[1] | split("."))[0], -# rowcount: .args.rowcount, -# krowpersec: (.args.rowcount / .elapsed[2] * 1000 * 60 + 0.5 | floor)}]' -# ["2017-03-16 21:20:24",10770,"cms_patient_dimension.sql",9,"\"DCONNOLLY\".patient_dimension", -# {"elapsed":"0:01:39","rowcount":419835,"krowpersec":252}] -# ["2017-03-16 21:22:12",10770,"cms_patient_dimension.sql",9,"\"DCONNOLLY\".patient_dimension", -# {"elapsed":"0:01:47","rowcount":419835,"krowpersec":234}] [loggers] @@ -50,7 +38,7 @@ level=INFO class=FileHandler # use detail_log_dir rather than dir to facilitate stream editing detail_log_dir=log -detail_log_file=%(detail_log_dir)s/grouse-detail.json +detail_log_file=%(detail_log_dir)s/cdm-detail.json # append to log file #args=('%(detail_log_file)s', 'a', None, True) # overwrite log file @@ -61,7 +49,7 @@ formatter=json level=DEBUG class=FileHandler luigi_debug_log_dir=log -luigi_debug_log_file=%(luigi_debug_log_dir)s/grouse-luigi-debug.json +luigi_debug_log_file=%(luigi_debug_log_dir)s/cdm-luigi-debug.json # append to log file # args=('%(luigi_debug_log_file)s', 'a') # overwrite log file From b2a9ff0d00a313654971aa72c8ef503dd332084c Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 14 Feb 2018 15:24:05 -0600 Subject: [PATCH 319/507] Updates to support Luigi task in dev environment. --- Oracle/PCORNetInit.sql | 2 +- Oracle/PCORNetLoader_ora.sql | 2 +- Oracle/med_admin.sql | 29 ++++++++++++++++++++++++++--- i2p_tasks.py | 12 ++++++------ requirements.txt | 5 +++-- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index b38a65e..88f6f10 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -1091,4 +1091,4 @@ CREATE TABLE provider( / -SELECT * FROM dual where rownum < 2 +SELECT 0 FROM dual diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 1fd6721..8a28e78 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1290,4 +1290,4 @@ begin end PCORNetLoader; / -SELECT * FROM dual where rownum < 2 \ No newline at end of file +SELECT 1 FROM HARVEST \ No newline at end of file diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 7831914..b0e1e9f 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -1,7 +1,30 @@ -------------------------------------------------------------------------------- +-- HELPER FUNCTIONS AND PROCEDURES +-------------------------------------------------------------------------------- +--These helper functions also exist in PCORNetInit.sql. They are reproduced +--here to make the med_admin script an independent luigi job. +--TODO: consolidate helpers in a single, table independent job. +create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS + BEGIN + DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. + tabname => table_name, + estimate_percent => 50, -- Percentage picked somewhat arbitrarily + cascade => TRUE, + degree => 16 + ); +END GATHER_TABLE_STATS; +/ +create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS + BEGIN + EXECUTE IMMEDIATE sqlstring; + EXCEPTION + WHEN OTHERS THEN NULL; +END PMN_DROPSQL; +/ +-------------------------------------------------------------------------------- -- MED_ADMIN -------------------------------------------------------------------------------- - BEGIN PMN_DROPSQL('DROP TABLE med_admin'); END; @@ -82,7 +105,7 @@ select med_start.patient_num, med_start.encounter_num, med_start.provider_id, me to_char(med_start.end_date, 'HH24:MI'), 'RX', med_p.pcori_basecode, med_dose.nval_num, med_dose.units_cd, 'OD', med_p.c_name, med_start.concept_cd, med_dose.nval_num, med_dose.units_cd, med_start.modifier_cd from med_start -left join pcornet_cdm.observation_fact_meds med_dose +left join pcornet_cdm.observation_fact med_dose on med_dose.instance_num = med_start.instance_num and med_dose.start_date = med_start.start_date and (med_dose.modifier_cd = 'MedObs:MAR_Dose|puff' @@ -114,4 +137,4 @@ BEGIN PCORNetMedAdmin(); END; / -SELECT count(*) from med_admin \ No newline at end of file +SELECT count(MEDADMINID) from med_admin where rownum = 1 \ No newline at end of file diff --git a/i2p_tasks.py b/i2p_tasks.py index 805d696..1b8157c 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -20,14 +20,14 @@ class PCORNetLoader_ora(SqlScriptTask): def variables(self) -> Environment: return dict(i2b2_meta_schema='BLUEHERONMETADATA', network_id='todo1', network_name='todo2', enrollment_months_back='2') + def requires(self) -> List[luigi.Task]: + '''Wrap each of `self.script.deps()` in a SqlScriptTask. + ''' + return [PCORNetInit()] + class PCORNetMed_Admin(SqlScriptTask): script = Script.med_admin @property def variables(self) -> Environment: - return dict(i2b2_meta_schema='BLUEHERONMETADATA', network_id='todo1', network_name='todo2', enrollment_months_back='2') - - def requires(self) -> List[luigi.Task]: - '''Wrap each of `self.script.deps()` in a SqlScriptTask. - ''' - return [PCORNetInit(), PCORNetLoader_ora()] \ No newline at end of file + return dict() diff --git a/requirements.txt b/requirements.txt index 4dd7554..5a42179 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ #Changes from 5 to 6.0.3 have created a twophase incompatibility between #SQLAlchemy and cx-Oracle. -cx-Oracle==6.0.3 +cx_Oracle==6.0.3 # Cython==0.25.1 # docutils==0.12 @@ -46,7 +46,8 @@ pandas==0.19.1 # pytz==2016.10 # requests==2.12.3 # six==1.10.0 -SQLAlchemy==1.1.4 +# SQLAlchemy==1.1.4 has a twophase conflict with cx_Oracle==6.0.3 +SQLAlchemy==1.2.1 #tornado==4.4.2 #XlsxWriter==0.9.4 From d1ae1a9efe18ae5655c89bc6a661cb7ef22c0609 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 14 Feb 2018 15:28:31 -0600 Subject: [PATCH 320/507] Updates to support Luigi task in dev environment. --- Oracle/med_admin.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index b0e1e9f..74acf13 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -78,7 +78,7 @@ raw_medadmin_code, raw_medadmin_dose_admin, raw_medadmin_dose_admin_unit, raw_me with med_start as ( select patient_num, encounter_num, provider_id, start_date, end_date, concept_cd, modifier_cd, instance_num - from pcornet_cdm.observation_fact_meds + from pcornet_cdm.observation_fact where modifier_cd = 'MedObs|MAR:New Bag' or modifier_cd = 'MedObs|MAR:Downtime Given - New Bag' or modifier_cd = 'MedObs|MAR:Given - Without Order' From b6d32d2745b9c1adc79e7087e5048bf483d1f392 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 14 Feb 2018 15:47:18 -0600 Subject: [PATCH 321/507] Updates to support Luigi task in dev environment. --- Oracle/med_admin.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 74acf13..f029de3 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -78,7 +78,7 @@ raw_medadmin_code, raw_medadmin_dose_admin, raw_medadmin_dose_admin_unit, raw_me with med_start as ( select patient_num, encounter_num, provider_id, start_date, end_date, concept_cd, modifier_cd, instance_num - from pcornet_cdm.observation_fact + from BLUEHERONDATA.observation_fact where modifier_cd = 'MedObs|MAR:New Bag' or modifier_cd = 'MedObs|MAR:Downtime Given - New Bag' or modifier_cd = 'MedObs|MAR:Given - Without Order' @@ -105,7 +105,7 @@ select med_start.patient_num, med_start.encounter_num, med_start.provider_id, me to_char(med_start.end_date, 'HH24:MI'), 'RX', med_p.pcori_basecode, med_dose.nval_num, med_dose.units_cd, 'OD', med_p.c_name, med_start.concept_cd, med_dose.nval_num, med_dose.units_cd, med_start.modifier_cd from med_start -left join pcornet_cdm.observation_fact med_dose +left join BLUEHERONDATA.observation_fact med_dose on med_dose.instance_num = med_start.instance_num and med_dose.start_date = med_start.start_date and (med_dose.modifier_cd = 'MedObs:MAR_Dose|puff' From ea87d1294a3bdb4e2fc3063320c15008bea2ec6a Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 14 Feb 2018 16:32:50 -0600 Subject: [PATCH 322/507] Updates to support Luigi task in dev environment. --- Oracle/med_admin_init.sql | 2 ++ i2p_tasks.py | 11 +++++++++++ script_lib.py | 2 ++ 3 files changed, 15 insertions(+) create mode 100644 Oracle/med_admin_init.sql diff --git a/Oracle/med_admin_init.sql b/Oracle/med_admin_init.sql new file mode 100644 index 0000000..6750d07 --- /dev/null +++ b/Oracle/med_admin_init.sql @@ -0,0 +1,2 @@ +/*Verify that there is useful data in the observation_fact table. If not, check for failed Heron ETL.*/ +SELECT count(patient_num) from BLUEHERONDATA.observation_fact where modifier_cd = 'MedObs|MAR:Given' and rownum = 1; \ No newline at end of file diff --git a/i2p_tasks.py b/i2p_tasks.py index 1b8157c..2d10e1e 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -31,3 +31,14 @@ class PCORNetMed_Admin(SqlScriptTask): @property def variables(self) -> Environment: return dict() + + def requires(self) -> List[luigi.Task]: + + return [PCORNetMed_Admin_Init()] + +class PCORNetMed_Admin_Init(SqlScriptTask): + script = Script.med_admin_init + + @property + def variables(self) -> Environment: + return dict() \ No newline at end of file diff --git a/script_lib.py b/script_lib.py index a066781..92147b5 100644 --- a/script_lib.py +++ b/script_lib.py @@ -271,6 +271,7 @@ class Script(ScriptMixin, enum.Enum): epic_flowsheets_transform, etl_tests_init, med_admin, + med_admin_init, migrate_fact_upload, ] = [ pkg.resource_string(__name__, @@ -282,6 +283,7 @@ class Script(ScriptMixin, enum.Enum): 'epic_flowsheets_transform.sql', 'etl_tests_init.sql', 'med_admin.sql', + 'med_admin_init.sql', 'migrate_fact_upload.sql', ] ] From be894946c5b9b58fc20de5994cc08892f83a2d5b Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 15 Feb 2018 13:16:30 -0600 Subject: [PATCH 323/507] Added luigi return codes to configuration --- client.cfg | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/client.cfg b/client.cfg index 1a925da..2283c0e 100644 --- a/client.cfg +++ b/client.cfg @@ -56,3 +56,14 @@ project_id = BlueHeron [resources] encounter_mapping=1 patient_mapping=1 + +[retcode] +# see also http://luigi.readthedocs.io/en/stable/configuration.html +# The following return codes are the recommended exit codes for Luigi +# They are in increasing level of severity (for most applications) +already_running=10 +missing_data=20 +not_run=25 +task_failed=30 +scheduling_error=35 +unhandled_exception=40 \ No newline at end of file From 778473824fd7ccb6aa799183c44bfed6d015661a Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 19 Feb 2018 12:55:26 -0600 Subject: [PATCH 324/507] CDM as separate Luigi tasks (in progress). Med_Admin stand alone. --- Oracle/PCORNetDrop.sql | 40 ++++++ Oracle/condition.sql | 108 +++++++++++++++ Oracle/death.sql | 53 ++++++++ Oracle/death_cause.sql | 19 +++ Oracle/demographic.sql | 146 +++++++++++++++++++++ Oracle/diagnosis.sql | 251 +++++++++++++++++++++++++++++++++++ Oracle/dispensing.sql | 184 ++++++++++++++++++++++++++ Oracle/encounter.sql | 105 +++++++++++++++ Oracle/enrollment.sql | 56 ++++++++ Oracle/harvest.sql | 82 ++++++++++++ Oracle/lab_result_cm.sql | 243 ++++++++++++++++++++++++++++++++++ Oracle/med_admin_init.sql | 2 +- Oracle/obs_clin.sql | 31 +++++ Oracle/obs_gen.sql | 31 +++++ Oracle/pcornet_init.sql | 219 +++++++++++++++++++++++++++++++ Oracle/pcornet_loader.sql | 1 + Oracle/pcornet_trail.sql | 20 +++ Oracle/prescribing.sql | 270 ++++++++++++++++++++++++++++++++++++++ Oracle/pro_cm.sql | 40 ++++++ Oracle/procedures.sql | 60 +++++++++ Oracle/provider.sql | 46 +++++++ Oracle/vital.sql | 140 ++++++++++++++++++++ etl_tasks.py | 4 +- i2p_tasks.py | 136 ++++++++++++++++--- script_lib.py | 52 ++++++-- 25 files changed, 2308 insertions(+), 31 deletions(-) create mode 100644 Oracle/PCORNetDrop.sql create mode 100644 Oracle/condition.sql create mode 100644 Oracle/death.sql create mode 100644 Oracle/death_cause.sql create mode 100644 Oracle/demographic.sql create mode 100644 Oracle/diagnosis.sql create mode 100644 Oracle/dispensing.sql create mode 100644 Oracle/encounter.sql create mode 100644 Oracle/enrollment.sql create mode 100644 Oracle/harvest.sql create mode 100644 Oracle/lab_result_cm.sql create mode 100644 Oracle/obs_clin.sql create mode 100644 Oracle/obs_gen.sql create mode 100644 Oracle/pcornet_init.sql create mode 100644 Oracle/pcornet_loader.sql create mode 100644 Oracle/pcornet_trail.sql create mode 100644 Oracle/prescribing.sql create mode 100644 Oracle/pro_cm.sql create mode 100644 Oracle/procedures.sql create mode 100644 Oracle/provider.sql create mode 100644 Oracle/vital.sql diff --git a/Oracle/PCORNetDrop.sql b/Oracle/PCORNetDrop.sql new file mode 100644 index 0000000..39c1a1e --- /dev/null +++ b/Oracle/PCORNetDrop.sql @@ -0,0 +1,40 @@ +BEGIN +PMN_DROPSQL('DROP TABLE demographic'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE enrollment'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE encounter'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE drg'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE diagnosis'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE sourcefact'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE pdxfact'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE originfact'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE procedures'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE vital'); +END; +/ \ No newline at end of file diff --git a/Oracle/condition.sql b/Oracle/condition.sql new file mode 100644 index 0000000..bfe257e --- /dev/null +++ b/Oracle/condition.sql @@ -0,0 +1,108 @@ +-------------------------------------------------------------------------------- +-- CONDITION +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE condition'); +END; +/ +CREATE TABLE condition( + CONDITIONID varchar(19) primary key, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, + REPORT_DATE date NULL, + RESOLVE_DATE date NULL, + ONSET_DATE date NULL, + CONDITION_STATUS varchar(2) NULL, + CONDITION varchar(18) NOT NULL, + CONDITION_TYPE varchar(2) NOT NULL, + CONDITION_SOURCE varchar(2) NOT NULL, + RAW_CONDITION_STATUS varchar(2) NULL, + RAW_CONDITION varchar(18) NULL, + RAW_CONDITION_TYPE varchar(2) NULL, + RAW_CONDITION_SOURCE varchar(2) NULL +) +/ + +BEGIN +PMN_DROPSQL('DROP sequence condition_seq'); +END; +/ +create sequence condition_seq +/ + +create or replace trigger condition_trg +before insert on condition +for each row +begin + select condition_seq.nextval into :new.CONDITIONID from dual; +end; +/ + +BEGIN +PMN_DROPSQL('DROP TABLE sourcefact2'); +END; +/ + +CREATE TABLE SOURCEFACT2 ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + DXSOURCE VARCHAR2(50) NULL, + C_FULLNAME VARCHAR2(700) NOT NULL + ) +/ +create or replace procedure PCORNetCondition as +begin + +PMN_DROPSQL('drop index condition_idx'); +PMN_DROPSQL('drop index sourcefact2_idx'); + +execute immediate 'truncate table condition'; +execute immediate 'truncate table sourcefact2'; + +insert into sourcefact2 + select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname + from i2b2fact factline + inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + where dxsource.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\%'; + +execute immediate 'create index sourcefact2_idx on sourcefact2 (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('SOURCEFACT2'); + +create_error_table('CONDITION'); + +insert into condition (patid, encounterid, report_date, resolve_date, condition, condition_type, condition_status, condition_source) +select distinct factline.patient_num, min(factline.encounter_num) encounterid, min(factline.start_date) report_date, NVL(max(factline.end_date),null) resolve_date, diag.pcori_basecode, +SUBSTR(diag.c_fullname,18,2) condition_type, + NVL2(max(factline.end_date) , 'RS', 'NI') condition_status, -- Imputed so might not be entirely accurate + NVL(SUBSTR(max(dxsource),INSTR(max(dxsource), ':')+1,2),'NI') condition_source +from i2b2fact factline +inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num +inner join pcornet_diag diag on diag.c_basecode = factline.concept_cd + left outer join sourcefact2 sf +on factline.patient_num=sf.patient_num +and factline.encounter_num=sf.encounter_num +and factline.provider_id=sf.provider_id +and factline.concept_cd=sf.concept_Cd +and factline.start_date=sf.start_Date +where diag.c_fullname like '\PCORI\DIAGNOSIS\%' +and sf.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\%' +group by factline.patient_num, diag.pcori_basecode, diag.c_fullname +log errors into ERR$_CONDITION reject limit unlimited +; + +execute immediate 'create index condition_idx on condition (PATID, ENCOUNTERID)'; +--GATHER_TABLE_STATS('CONDITION'); + +end PCORNetCondition; +/ +BEGIN +PCORNetCondition(); +END; +/ +SELECT count(CONDITIONID) from condition where rownum = 1 +--SELECT 1 FROM dual \ No newline at end of file diff --git a/Oracle/death.sql b/Oracle/death.sql new file mode 100644 index 0000000..da948d4 --- /dev/null +++ b/Oracle/death.sql @@ -0,0 +1,53 @@ +-------------------------------------------------------------------------------- +-- DEATH +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE death'); +END; +/ +CREATE TABLE death( + PATID varchar(50) NOT NULL, + DEATH_DATE date NOT NULL, + DEATH_DATE_IMPUTE varchar(2) NULL, + DEATH_SOURCE varchar(2) NOT NULL, + DEATH_MATCH_CONFIDENCE varchar(2) NULL +) +/ +create or replace procedure PCORNetDeath as +begin + +execute immediate 'truncate table death'; + +insert into death( patid, death_date, death_date_impute, death_source, death_match_confidence) +select distinct pat.patient_num, pat.death_date, +case when vital_status_cd like 'X%' then 'B' + when vital_status_cd like 'M%' then 'D' + when vital_status_cd like 'Y%' then 'N' + else 'OT' + end death_date_impute, + 'NI' death_source, + 'NI' death_match_confidence +from ( + /* KUMC specific fix to address unknown death dates */ + select + ibp.patient_num, + case when ibf.concept_cd is not null then DATE '2100-12-31' -- in accordance with the CDM v3 spec + else ibp.death_date end death_date, + case when ibf.concept_cd is not null then 'OT' + else upper(ibp.vital_status_cd) end vital_status_cd + from i2b2patient ibp + left join i2b2fact ibf + on ibp.patient_num=ibf.patient_num + and ibf.CONCEPT_CD='DEM|VITAL:yu' +) pat +where (pat.death_date is not null or vital_status_cd like 'Z%') and pat.patient_num in (select patid from demographic); + +end; +/ +BEGIN +PCORNetDeath(); +END; +/ +--SELECT count(PATID) from death where rownum = 1 +SELECT 1 FROM dual \ No newline at end of file diff --git a/Oracle/death_cause.sql b/Oracle/death_cause.sql new file mode 100644 index 0000000..40c14b7 --- /dev/null +++ b/Oracle/death_cause.sql @@ -0,0 +1,19 @@ +-------------------------------------------------------------------------------- +-- DEATH_CAUSE +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE death_cause'); +END; +/ +CREATE TABLE death_cause( + PATID varchar(50) NOT NULL, + DEATH_CAUSE varchar(8) NOT NULL, + DEATH_CAUSE_CODE varchar(2) NOT NULL, + DEATH_CAUSE_TYPE varchar(2) NOT NULL, + DEATH_CAUSE_SOURCE varchar(2) NOT NULL, + DEATH_CAUSE_CONFIDENCE varchar(2) NULL +) +/ +--SELECT count(PATID) from death_cause where rownum = 1 +SELECT count(MEDADMINID) from med_admin where rownum = 1 \ No newline at end of file diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql new file mode 100644 index 0000000..72100c0 --- /dev/null +++ b/Oracle/demographic.sql @@ -0,0 +1,146 @@ +-------------------------------------------------------------------------------- +-- DEMOGRAPHIC +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE demographic'); +END; +/ +CREATE TABLE demographic( + PATID varchar(50) NOT NULL, + BIRTH_DATE date NULL, + BIRTH_TIME varchar(5) NULL, + SEX varchar(2) NULL, + SEXUAL_ORIENTATION varchar(2) NULL, + GENDER_IDENTITY varchar(2) NULL, + HISPANIC varchar(2) NULL, + BIOBANK_FLAG varchar(1) DEFAULT 'N', + RACE varchar(2) NULL, + RAW_SEX varchar(50) NULL, + RAW_SEXUAL_ORIENTATION varchar(50) NULL, + RAW_GENDER_IDENTITY varchar(50) NULL, + RAW_HISPANIC varchar(50) NULL, + RAW_RACE varchar(50) NULL +) +/ +create or replace procedure PCORNetDemographic as + +sqltext varchar2(4000); +cursor getsql is +--1 -- S,R,NH + select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) + select '1', patient_num, birth_date, to_char(birth_date,''HH24:MI''), sex.pcori_basecode, 'NI', 'NI', 'NI', race.pcori_basecode + from i2b2patient p + where lower(p.sex_cd) in (lower(sex.c_dimcode)) + and lower(p.race_cd) in (lower(race.c_dimcode)) + and lower(nvl(p.ethnicity_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') + from pcornet_demo race, pcornet_demo sex + where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' + and race.c_visualattributes like 'L%' + and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' + and sex.c_visualattributes like 'L%' +union -- A - S,R,H + select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) + select 'A', patient_num, birth_date, to_char(birth_date,''HH24:MI''), sex.pcori_basecode, 'NI', 'NI', hisp.pcori_basecode, race.pcori_basecode + from i2b2patient p + where lower(p.sex_cd) in (lower(sex.c_dimcode)) + and lower(p.race_cd) in (lower(race.c_dimcode)) + and lower(p.ethnicity_cd) in (lower(hisp.c_dimcode)) + from pcornet_demo race, pcornet_demo hisp, pcornet_demo sex + where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' + and race.c_visualattributes like 'L%' + and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' + and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' + and sex.c_visualattributes like 'L%' +union --2 S, nR, nH + select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) + select '2', patient_num, birth_date, to_char(birth_date,''HH24:MI''), sex.pcori_basecode, 'NI', 'NI', 'NI', 'NI' + from i2b2patient p + where lower(nvl(p.sex_cd,''xx'')) in (lower(sex.c_dimcode)) + and lower(nvl(p.race_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='RACE') + and lower(nvl(p.ethnicity_cd,'ni')) not in (select lower(code) from pcornet_codelist where codetype='HISPANIC') + from pcornet_demo sex + where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' + and sex.c_visualattributes like 'L%' +union --3 -- nS,R, NH + select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) + select '3', patient_num, birth_date, to_char(birth_date,''HH24:MI''), 'NI', 'NI', 'NI', 'NI', race.pcori_basecode + from i2b2patient p + where lower(nvl(p.sex_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='SEX') + and lower(p.race_cd) in (lower(race.c_dimcode)) + and lower(nvl(p.ethnicity_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='HISPANIC') + from pcornet_demo race + where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' + and race.c_visualattributes like 'L%' +union --B -- nS,R, H + select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) + select 'B', patient_num, birth_date, to_char(birth_date,'HH24:MI'), 'NI', 'NI', 'NI', hisp.pcori_basecode, race.pcori_basecode + from i2b2patient p + where lower(nvl(p.sex_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='SEX') + and lower(p.race_cd) in (lower(race.c_dimcode)) + and lower(p.ethnicity_cd) in (lower(hisp.c_dimcode)) + from pcornet_demo race,pcornet_demo hisp + where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' + and race.c_visualattributes like 'L%' + and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' +union --4 -- S, NR, H + select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) + select '4', patient_num, birth_date, to_char(birth_date,''HH24:MI''), sex.pcori_basecode, 'NI', 'NI', hisp.pcori_basecode, 'NI' + from i2b2patient p + where lower(nvl(p.sex_cd,'NI')) in (lower(sex.c_dimcode)) + and lower(nvl(p.ethnicity_cd,'NI')) in (lower(hisp.c_dimcode)) + and lower(nvl(p.race_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='RACE') + from pcornet_demo sex, pcornet_demo hisp + where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' + and sex.c_visualattributes like 'L%' + and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' +union --5 -- NS, NR, H + select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) + select '5', patient_num, birth_date, to_char(birth_date,'HH24:MI'), 'NI', 'NI', 'NI', hisp.pcori_basecode, 'NI' + from i2b2patient p + where lower(nvl(p.sex_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='SEX') + and lower(nvl(p.race_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='RACE') + and lower(nvl(p.ethnicity_cd,'NI')) in (lower(hisp.c_dimcode)) + from pcornet_demo hisp + where hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' +union --6 -- NS, NR, nH + select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) + select '6', patient_num, birth_date, to_char(birth_date,'HH24:MI'), 'NI', 'NI', 'NI', 'NI', 'NI' + from i2b2patient p + where lower(nvl(p.sex_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='SEX') + and lower(nvl(p.ethnicity_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='HISPANIC') + and lower(nvl(p.race_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='RACE') + from dual; + +begin +pcornet_popcodelist; + +PMN_DROPSQL('drop index demographic_pk'); + +execute immediate 'truncate table demographic'; + +OPEN getsql; +LOOP +FETCH getsql INTO sqltext; + EXIT WHEN getsql%NOTFOUND; +-- insert into st values (sqltext); + execute immediate sqltext; + COMMIT; +END LOOP; +CLOSE getsql; + +execute immediate 'create unique index demographic_pk on demographic (PATID)'; +--GATHER_TABLE_STATS('DEMOGRAPHIC'); + +end PCORNetDemographic; +/ +BEGIN +PCORNetDemographic(); +END; +/ +--SELECT count(PATID) from demographic where rownum = 1 +SELECT count(MEDADMINID) from med_admin where rownum = 1 \ No newline at end of file diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql new file mode 100644 index 0000000..8b15c9f --- /dev/null +++ b/Oracle/diagnosis.sql @@ -0,0 +1,251 @@ +-------------------------------------------------------------------------------- +-- DIAGNOSIS +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE diagnosis'); +END; +/ +CREATE TABLE diagnosis( + DIAGNOSISID varchar(19) primary key, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NOT NULL, + ENC_TYPE varchar(2) NULL, + ADMIT_DATE date NULL, + PROVIDERID varchar(50) NULL, + DX varchar(18) NOT NULL, + DX_TYPE varchar(2) NOT NULL, + DX_SOURCE varchar(2) NOT NULL, + DX_ORIGIN varchar(2) NULL, + PDX varchar(2) NULL, + RAW_DX varchar(50) NULL, + RAW_DX_TYPE varchar(50) NULL, + RAW_DX_SOURCE varchar(50) NULL, + RAW_ORIGDX varchar(50) NULL, + RAW_PDX varchar(50) NULL +) +/ + +BEGIN +PMN_DROPSQL('DROP sequence diagnosis_seq'); +END; +/ +create sequence diagnosis_seq +/ + +create or replace trigger diagnosis_trg +before insert on diagnosis +for each row +begin + select diagnosis_seq.nextval into :new.DIAGNOSISID from dual; +end; +/ + +BEGIN +PMN_DROPSQL('DROP TABLE sourcefact'); +END; +/ + +CREATE TABLE SOURCEFACT ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + DXSOURCE VARCHAR2(50) NULL, + C_FULLNAME VARCHAR2(700) NOT NULL + ) +/ + +BEGIN +PMN_DROPSQL('DROP TABLE pdxfact'); +END; +/ + +CREATE TABLE PDXFACT ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + PDXSOURCE VARCHAR2(50) NULL, + C_FULLNAME VARCHAR2(700) NOT NULL + ) +/ + +BEGIN +PMN_DROPSQL('DROP TABLE originfact'); +END; +/ + +CREATE TABLE ORIGINFACT ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + ORIGINSOURCE VARCHAR2(50) NULL, + C_FULLNAME VARCHAR2(700) NOT NULL + ) +/ +create or replace procedure PCORNetDiagnosis as +begin + +PMN_DROPSQL('drop index diagnosis_idx'); +PMN_DROPSQL('drop index sourcefact_idx'); +PMN_DROPSQL('drop index pdxfact_idx'); +PMN_DROPSQL('drop index originfact_idx'); + +execute immediate 'truncate table diagnosis'; +execute immediate 'truncate table sourcefact'; +execute immediate 'truncate table pdxfact'; +execute immediate 'truncate table originfact'; + +insert into sourcefact + select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname + from i2b2fact factline + inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + where dxsource.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\%'; + +execute immediate 'create index sourcefact_idx on sourcefact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('SOURCEFACT'); + +insert into pdxfact + select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode pdxsource,dxsource.c_fullname + from i2b2fact factline + inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + and dxsource.c_fullname like '\PCORI_MOD\PDX\%'; + +execute immediate 'create index pdxfact_idx on pdxfact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('PDXFACT'); + +insert into originfact --CDM 3.1 addition + select patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode originsource, dxsource.c_fullname + from i2b2fact factline + inner join ENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + and dxsource.c_fullname like '\PCORI_MOD\DX_ORIGIN\%'; + +execute immediate 'create index originfact_idx on originfact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('ORIGINFACT'); + +insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, dx_origin, pdx) +/* KUMC started billing with ICD10 on Oct 1, 2015. */ +with icd10_transition as ( + select date '2015-10-01' as cutoff from dual +) + +/* Encoding of DX type from the CDM 3.1 Specification + http://pcornet.org/wp-content/uploads/2016/11/2016-11-15-PCORnet-Common-Data-Model-v3.1_Specification.pdf + */ +, dx_type as ( + select '09' as icd_9_cm + , '10' as icd_10_cm + , '11' as icd_11_cm + , 'SM' as snomed_ct + , 'NI' as no_info + , 'UN' as unknown + , 'OT' as other + from dual) + +/* DX_IDs may have mappings to both ICD9 and ICD10 */ +, has9 as ( + select distinct c_basecode, pcori_basecode icd9_code + from "&&i2b2_meta_schema".pcornet_diag diag + where diag.c_fullname like '\PCORI\DIAGNOSIS\09%' + and pcori_basecode is not null +) +, has10 as ( + select distinct c_basecode, pcori_basecode icd10_code + from "&&i2b2_meta_schema".pcornet_diag diag + where diag.c_fullname like '\PCORI\DIAGNOSIS\10%' + and pcori_basecode is not null +) +, diag as ( + select distinct diag.c_basecode, diag.pcori_basecode, has9.icd9_code, has10.icd10_code + , case when diag.pcori_basecode = has9.icd9_code then (select icd_9_cm from dx_type) + when diag.pcori_basecode = has10.icd10_code then (select icd_10_cm from dx_type) + else (select no_info from dx_type) + end dx_type + from pcornet_diag diag + left join has9 on has9.c_basecode = diag.c_basecode + left join has10 on has10.c_basecode = diag.c_basecode + + where diag.c_fullname like '\PCORI\DIAGNOSIS\%' + and diag.pcori_basecode is not null +) +/* Convert i2b2 diagnosis facts to pcori diagnosis facts + * Note: The mapping of i2b2 diagnosis facts to pcori diagnosis facts + * with respect to ICD9 and ICD10 is not 1-to-1 (there may exist an i2b2 + * diagnosis fact that could be represented as both ICD9 and ICD10 pcori fact) + * + * To prevent an increase in the number of post conversion facts, prioritize + * filtering ICD10s prior to 1/1/2015 and ICD9s after 1/1/2015 if resulting + * facts produce duplicates based on the i2b2 fact primary key. + */ +, diag_fact_merge as ( + select factline.*, diag.* + , row_number() over (partition by factline.patient_num + , factline.encounter_num + , factline.concept_cd + , factline.start_date + , factline.modifier_cd + , factline.instance_num + order by (case when diag.dx_type = (select icd_9_cm from dx_type) + and factline.start_date >= (select cutoff from icd10_transition) then 0 + when diag.dx_type = (select icd_10_cm from dx_type) + and factline.start_date < (select cutoff from icd10_transition) then 0 + else 1 + end) asc) unique_row + from i2b2fact factline + join diag on diag.c_basecode = factline.concept_cd +) +, diag_fact_cutoff_filter as ( + select * from diag_fact_merge where unique_row = 1 +) + +select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, enc.admit_date, enc.providerid + , factline.pcori_basecode dx + , factline.dx_type dxtype, + CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END dx_source, + nvl(SUBSTR(originsource,INSTR(originsource, ':')+1,2),'NI') dx_origin, + CASE WHEN enc_type in ('EI', 'IP', 'IS') -- PDX is "relevant only on IP and IS encounters" + THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, ':')+1,2),'NI') + ELSE 'X' END PDX +from diag_fact_cutoff_filter factline +inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + left outer join sourcefact +on factline.patient_num=sourcefact.patient_num +and factline.encounter_num=sourcefact.encounter_num +and factline.provider_id=sourcefact.provider_id +and factline.concept_cd=sourcefact.concept_Cd +and factline.start_date=sourcefact.start_Date +left outer join pdxfact +on factline.patient_num=pdxfact.patient_num +and factline.encounter_num=pdxfact.encounter_num +and factline.provider_id=pdxfact.provider_id +and factline.concept_cd=pdxfact.concept_cd +and factline.start_date=pdxfact.start_Date +left outer join originfact --CDM 3.1 addition +on factline.patient_num=originfact.patient_num +and factline.encounter_num=originfact.encounter_num +and factline.provider_id=originfact.provider_id +and factline.concept_cd=originfact.concept_cd +and factline.start_date=originfact.start_Date +where (sourcefact.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\%' or sourcefact.c_fullname is null) +-- order by enc.admit_date desc +; + +execute immediate 'create index diagnosis_idx on diagnosis (PATID, ENCOUNTERID)'; +--GATHER_TABLE_STATS('DIAGNOSIS'); + +end PCORNetDiagnosis; +/ +BEGIN +PCORNetDiagnosis(); +END; +/ +SELECT count(DIAGNOSISID) from diagnosis where rownum = 1 +--SELECT 1 FROM dual \ No newline at end of file diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql new file mode 100644 index 0000000..99e8de9 --- /dev/null +++ b/Oracle/dispensing.sql @@ -0,0 +1,184 @@ +-------------------------------------------------------------------------------- +-- DISPENSING +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE dispensing'); +END; +/ +CREATE TABLE dispensing( + DISPENSINGID varchar(19) primary key, + PATID varchar(50) NOT NULL, + PRESCRIBINGID varchar(19) NULL, + DISPENSE_DATE date NOT NULL, + NDC varchar (11) NOT NULL, + DISPENSE_SUP number(18) NULL, + DISPENSE_AMT number(18) NULL, + RAW_NDC varchar (50) NULL +) +/ + +BEGIN +PMN_DROPSQL('DROP sequence dispensing_seq'); +END; +/ +create sequence dispensing_seq +/ + +create or replace trigger dispensing_trg +before insert on dispensing +for each row +begin + select dispensing_seq.nextval into :new.DISPENSINGID from dual; +end; +/ + +BEGIN +PMN_DROPSQL('DROP TABLE DISP_SUPPLY'); --Changed the table 'supply' to the name 'disp_supply' to avoid conflicts with the prescribing procedure, Matthew Joss 8/16/16 +END; +/ + +CREATE TABLE DISP_SUPPLY ( + NVAL_NUM NUMBER(18,5) NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL + ) +/ + +BEGIN +PMN_DROPSQL('DROP TABLE amount'); +END; +/ + +CREATE TABLE AMOUNT ( + NVAL_NUM NUMBER(18,5) NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL + ) +/ +create or replace procedure PCORNetDispensing as +begin + +PMN_DROPSQL('drop index dispensing_idx'); + +execute immediate 'truncate table dispensing'; +/* +PMN_DROPSQL('DROP TABLE supply'); +sqltext := 'create table supply as '|| +'(select nval_num,encounter_num,concept_cd from i2b2fact supply '|| +' inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| +' join pcornet_med supplycode '|| +' on supply.modifier_cd = supplycode.c_basecode '|| +' and supplycode.c_fullname like ''\PCORI_MOD\RX_DAYS_SUPPLY\'' ) '; +PMN_EXECUATESQL(sqltext); + + +PMN_DROPSQL('DROP TABLE amount'); +sqltext := 'create table amount as '|| +'(select nval_num,encounter_num,concept_cd from i2b2fact amount '|| +' join pcornet_med amountcode '|| +' on amount.modifier_cd = amountcode.c_basecode '|| +' and amountcode.c_fullname like ''\PCORI_MOD\RX_QUANTITY\'') '; +PMN_EXECUATESQL(sqltext); +*/ + +/* NOTE: New transformation developed by KUMC */ + +insert into dispensing ( + PATID + ,PRESCRIBINGID + ,DISPENSE_DATE -- using start_date from i2b2 + ,NDC --using pcornet_med pcori_ndc - new column! + ,DISPENSE_SUP ---- modifier nval_num + ,DISPENSE_AMT -- modifier nval_num +-- ,RAW_NDC +) +/* Below is the Cycle 2 fix for populating the DISPENSING table */ +with disp_status as ( + select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd + from i2b2fact ibf + join "&&i2b2_meta_schema".pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode + where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' + /* TODO: Generalize for other sites. The '< 12' makes sure only 11 digit + codes are included. */ + and length(replace(ibf.concept_cd, 'NDC:', '')) < 12 +) +, disp_quantity as ( + select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num + from i2b2fact ibf + join "&&i2b2_meta_schema".pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode + where pnm.c_fullname like '\PCORI_MOD\RX_QUANTITY\%' +) +, disp_supply as ( + select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num + from i2b2fact ibf + join "&&i2b2_meta_schema".pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode + where pnm.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\%' +) +select distinct + st.patient_num patid, + null prescribingid, + st.start_date dispense_date, + replace(st.concept_cd, 'NDC:', '') ndc, -- TODO: Generalize this for other sites. + ds.nval_num dispense_sup, + qt.nval_num dispense_amt +from disp_status st +left outer join disp_quantity qt + on st.patient_num=qt.patient_num + and st.encounter_num=qt.encounter_num + and st.concept_cd=qt.concept_cd + and st.instance_num=qt.instance_num + and st.start_date=qt.start_date +left outer join disp_supply ds + on st.patient_num=ds.patient_num + and st.encounter_num=ds.encounter_num + and st.concept_cd=ds.concept_cd + and st.instance_num=ds.instance_num + and st.start_date=ds.start_date +; + +/* NOTE: The original SCILHS transformation is below. + +-- insert data with outer joins to ensure all records are included even if some data elements are missing + +select m.patient_num, null,m.start_date, NVL(mo.pcori_ndc,'NA') + ,max(supply.nval_num) sup, max(amount.nval_num) amt +from i2b2fact m inner join pcornet_med mo +on m.concept_cd = mo.c_basecode +inner join encounter enc on enc.encounterid = m.encounter_Num + + -- jgk bugfix 11/2 - we weren't filtering dispensing events + inner join (select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2fact basis + inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num + join pcornet_med basiscode + on basis.modifier_cd = basiscode.c_basecode + and basiscode.c_fullname='\PCORI_MOD\RX_BASIS\DI\') basis + on m.encounter_num = basis.encounter_num + and m.concept_cd = basis.concept_Cd + + left join supply + on m.encounter_num = supply.encounter_num + and m.concept_cd = supply.concept_Cd + + + left join amount + on m.encounter_num = amount.encounter_num + and m.concept_cd = amount.concept_Cd + +group by m.encounter_num ,m.patient_num, m.start_date, mo.pcori_ndc; +*/ + +execute immediate 'create index dispensing_idx on dispensing (PATID)'; +--GATHER_TABLE_STATS('DISPENSING'); + +end PCORNetDispensing; +/ +BEGIN +PCORNetDispensing(); +END; +/ +SELECT count(DISPENSINGID) from dispensing where rownum = 1 +--SELECT 1 FROM dual \ No newline at end of file diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql new file mode 100644 index 0000000..fdfa4e4 --- /dev/null +++ b/Oracle/encounter.sql @@ -0,0 +1,105 @@ +-------------------------------------------------------------------------------- +-- ENCOUNTER +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE encounter'); +END; +/ +CREATE TABLE encounter( + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NOT NULL, + ADMIT_DATE date NULL, + ADMIT_TIME varchar(5) NULL, + DISCHARGE_DATE date NULL, + DISCHARGE_TIME varchar(5) NULL, + PROVIDERID varchar(50) NULL, + FACILITY_LOCATION varchar(3) NULL, + ENC_TYPE varchar(2) NOT NULL, + FACILITYID varchar(50) NULL, + DISCHARGE_DISPOSITION varchar(2) NULL, + DISCHARGE_STATUS varchar(2) NULL, + DRG varchar(3) NULL, + DRG_TYPE varchar(2) NULL, + ADMITTING_SOURCE varchar(2) NULL, + RAW_SITEID varchar (50) NULL, + RAW_ENC_TYPE varchar(50) NULL, + RAW_DISCHARGE_DISPOSITION varchar(50) NULL, + RAW_DISCHARGE_STATUS varchar(50) NULL, + RAW_DRG_TYPE varchar(50) NULL, + RAW_ADMITTING_SOURCE varchar(50) NULL +) +/ +BEGIN +PMN_DROPSQL('DROP TABLE drg'); +END; +/ +CREATE TABLE drg ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + DRG_TYPE VARCHAR2(2), + DRG VARCHAR2(3), + RN NUMBER +) +/ +create or replace procedure PCORNetEncounter as +begin + +PMN_DROPSQL('drop index encounter_pk'); +PMN_DROPSQL('drop index encounter_idx'); +PMN_DROPSQL('drop index drg_idx'); + +execute immediate 'truncate table encounter'; +execute immediate 'truncate table drg'; + +insert into drg +select * from +(select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from +(select patient_num,encounter_num,drg_type,max(drg) drg from +(select distinct f.patient_num,encounter_num,SUBSTR(c_fullname,22,2) drg_type,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,3) drg from i2b2fact f +inner join demographic d on f.patient_num=d.patid +inner join pcornet_enc enc on enc.c_basecode = f.concept_cd +and enc.c_fullname like '\PCORI\ENCOUNTER\DRG\%') drg1 group by patient_num,encounter_num,drg_type) drg) drg +where rn=1; + +execute immediate 'create index drg_idx on drg (patient_num, encounter_num)'; +GATHER_TABLE_STATS('drg'); + +insert into encounter(PATID,ENCOUNTERID,admit_date ,ADMIT_TIME , + DISCHARGE_DATE ,DISCHARGE_TIME ,PROVIDERID ,FACILITY_LOCATION + ,ENC_TYPE ,FACILITYID ,DISCHARGE_DISPOSITION , + DISCHARGE_STATUS ,DRG ,DRG_TYPE ,ADMITTING_SOURCE) +select distinct v.patient_num, v.encounter_num, + start_Date, + to_char(start_Date,'HH24:MI'), + end_Date, + to_char(end_Date,'HH24:MI'), + providerid, + 'NI' location_zip, /* See TODO above */ +(case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, + 'NI' facility_id, /* See TODO above */ + CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END, + CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END, + drg.drg, drg_type, + CASE WHEN admitting_source IS NULL THEN 'NI' ELSE admitting_source END admitting_source +from i2b2visit v inner join demographic d on v.patient_num=d.patid +left outer join drg -- This section is bugfixed to only include 1 drg if multiple DRG types exist in a single encounter... + on drg.patient_num=v.patient_num and drg.encounter_num=v.encounter_num +left outer join +-- Encounter type. Note that this requires a full table scan on the ontology table, so it is not particularly efficient. +(select patient_num, encounter_num, inout_cd,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,2) pcori_enctype from i2b2visit v + inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype + on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num; + +execute immediate 'create unique index encounter_pk on encounter (ENCOUNTERID)'; +execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; +GATHER_TABLE_STATS('ENCOUNTER'); + +end PCORNetEncounter; +/ +BEGIN +PCORNetEncounter(); +END; +/ +--SELECT count(ENCOUNTERID) from encounter where rownum = 1 +SELECT 1 FROM dual \ No newline at end of file diff --git a/Oracle/enrollment.sql b/Oracle/enrollment.sql new file mode 100644 index 0000000..4d75693 --- /dev/null +++ b/Oracle/enrollment.sql @@ -0,0 +1,56 @@ +-------------------------------------------------------------------------------- +-- ENROLLMENT +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE enrollment'); +END; +/ +CREATE TABLE enrollment ( + PATID varchar(50) NOT NULL, + ENR_START_DATE date NOT NULL, + ENR_END_DATE date NULL, + CHART varchar(1) NULL, + ENR_BASIS varchar(1) NOT NULL, + RAW_CHART varchar(50) NULL, + RAW_BASIS varchar(50) NULL +) +/ +create or replace procedure PCORNetEnroll as +begin + +PMN_DROPSQL('drop index enrollment_idx'); + +execute immediate 'truncate table enrollment'; + +INSERT INTO enrollment(PATID, ENR_START_DATE, ENR_END_DATE, CHART, ENR_BASIS) +with pats_delta as ( + -- If only one visit, visit_delta_days will be 0 + select patient_num, max(start_date) - min(start_date) visit_delta_days + from i2b2visit + where start_date > add_months(sysdate, -&&enrollment_months_back) + group by patient_num + ), +enrolled as ( + select distinct patient_num + from pats_delta + where visit_delta_days > 30 + ) +select + visit.patient_num patid, min(visit.start_date) enr_start_date, + max(visit.start_date) enr_end_date, 'Y' chart, 'A' enr_basis +from enrolled enr +join i2b2visit visit on enr.patient_num = visit.patient_num +group by visit.patient_num; + +execute immediate 'create index enrollment_idx on enrollment (PATID)'; +GATHER_TABLE_STATS('ENROLLMENT'); + +end PCORNetEnroll; +/ +BEGIN +PCORNetEnroll(); +END; +/ +--SELECT count(PATID) from enrollment where rownum = 1 +SELECT 1 FROM dual \ No newline at end of file diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql new file mode 100644 index 0000000..dad37b8 --- /dev/null +++ b/Oracle/harvest.sql @@ -0,0 +1,82 @@ +-------------------------------------------------------------------------------- +-- HARVEST +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE harvest'); +END; +/ +CREATE TABLE harvest( + NETWORKID varchar(10) NOT NULL, + NETWORK_NAME varchar(20) NULL, + DATAMARTID varchar(10) NOT NULL, + DATAMART_NAME varchar(20) NULL, + DATAMART_PLATFORM varchar(2) NULL, + CDM_VERSION numeric(8, 2) NULL, + DATAMART_CLAIMS varchar(2) NULL, + DATAMART_EHR varchar(2) NULL, + BIRTH_DATE_MGMT varchar(2) NULL, + ENR_START_DATE_MGMT varchar(2) NULL, + ENR_END_DATE_MGMT varchar(2) NULL, + ADMIT_DATE_MGMT varchar(2) NULL, + DISCHARGE_DATE_MGMT varchar(2) NULL, + PX_DATE_MGMT varchar(2) NULL, + RX_ORDER_DATE_MGMT varchar(2) NULL, + RX_START_DATE_MGMT varchar(2) NULL, + RX_END_DATE_MGMT varchar(2) NULL, + DISPENSE_DATE_MGMT varchar(2) NULL, + LAB_ORDER_DATE_MGMT varchar(2) NULL, + SPECIMEN_DATE_MGMT varchar(2) NULL, + RESULT_DATE_MGMT varchar(2) NULL, + MEASURE_DATE_MGMT varchar(2) NULL, + ONSET_DATE_MGMT varchar(2) NULL, + REPORT_DATE_MGMT varchar(2) NULL, + RESOLVE_DATE_MGMT varchar(2) NULL, + PRO_DATE_MGMT varchar(2) NULL, + REFRESH_DEMOGRAPHIC_DATE date NULL, + REFRESH_ENROLLMENT_DATE date NULL, + REFRESH_ENCOUNTER_DATE date NULL, + REFRESH_DIAGNOSIS_DATE date NULL, + REFRESH_PROCEDURES_DATE date NULL, + REFRESH_VITAL_DATE date NULL, + REFRESH_DISPENSING_DATE date NULL, + REFRESH_LAB_RESULT_CM_DATE date NULL, + REFRESH_CONDITION_DATE date NULL, + REFRESH_PRO_CM_DATE date NULL, + REFRESH_PRESCRIBING_DATE date NULL, + REFRESH_PCORNET_TRIAL_DATE date NULL, + REFRESH_DEATH_DATE date NULL, + REFRESH_DEATH_CAUSE_DATE date NULL +) +/ +create or replace procedure PCORNetHarvest as +begin + +execute immediate 'truncate table harvest'; + +INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) + select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.PX_DATE_MGMT, hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, + case when (select count(*) from demographic) > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, + case when (select count(*) from enrollment) > 0 then current_date else null end REFRESH_ENROLLMENT_DATE, + case when (select count(*) from encounter) > 0 then current_date else null end REFRESH_ENCOUNTER_DATE, + case when (select count(*) from diagnosis) > 0 then current_date else null end REFRESH_DIAGNOSIS_DATE, + case when (select count(*) from procedures) > 0 then current_date else null end REFRESH_PROCEDURES_DATE, + case when (select count(*) from vital) > 0 then current_date else null end REFRESH_VITAL_DATE, + case when (select count(*) from dispensing) > 0 then current_date else null end REFRESH_DISPENSING_DATE, + case when (select count(*) from lab_result_cm) > 0 then current_date else null end REFRESH_LAB_RESULT_CM_DATE, + case when (select count(*) from condition) > 0 then current_date else null end REFRESH_CONDITION_DATE, + case when (select count(*) from pro_cm) > 0 then current_date else null end REFRESH_PRO_CM_DATE, + case when (select count(*) from prescribing) > 0 then current_date else null end REFRESH_PRESCRIBING_DATE, + case when (select count(*) from pcornet_trial) > 0 then current_date else null end REFRESH_PCORNET_TRIAL_DATE, + case when (select count(*) from death) > 0 then current_date else null end REFRESH_DEATH_DATE, + case when (select count(*) from death_cause) > 0 then current_date else null end REFRESH_DEATH_CAUSE_DATE + from harvest_local hl; + +end PCORNetHarvest; +/ +BEGIN +PCORNetHarvest(); +END; +/ +SELECT count(NETWORKID) from Harvest where rownum = 1 +--SELECT 1 FROM dual \ No newline at end of file diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql new file mode 100644 index 0000000..80efa99 --- /dev/null +++ b/Oracle/lab_result_cm.sql @@ -0,0 +1,243 @@ +-------------------------------------------------------------------------------- +-- LAB_RESULT_CM +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE lab_result_cm'); +END; +/ +CREATE TABLE lab_result_cm( + LAB_RESULT_CM_ID varchar(19) primary key, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, + LAB_NAME varchar(10) NULL, + SPECIMEN_SOURCE varchar(10) NULL, + LAB_LOINC varchar(10) NULL, + PRIORITY varchar(2) NULL, + RESULT_LOC varchar(2) NULL, + LAB_PX varchar(11) NULL, + LAB_PX_TYPE varchar(2) NULL, + LAB_ORDER_DATE date NULL, + SPECIMEN_DATE date NULL, + SPECIMEN_TIME varchar(5) NULL, + RESULT_DATE date NULL, + RESULT_TIME varchar(5) NULL, + RESULT_QUAL varchar(12) NULL, + RESULT_NUM number (15,8) NULL, + RESULT_MODIFIER varchar(2) NULL, + RESULT_UNIT varchar(11) NULL, + NORM_RANGE_LOW varchar(10) NULL, + NORM_MODIFIER_LOW varchar(2) NULL, + NORM_RANGE_HIGH varchar(10) NULL, + NORM_MODIFIER_HIGH varchar(2) NULL, + ABN_IND varchar(2) NULL, + RAW_LAB_NAME varchar(50) NULL, + RAW_LAB_CODE varchar(50) NULL, + RAW_PANEL varchar(50) NULL, + RAW_RESULT varchar(50) NULL, + RAW_UNIT varchar(50) NULL, + RAW_ORDER_DEPT varchar(50) NULL, + RAW_FACILITY_CODE varchar(50) NULL +) +/ + +BEGIN +PMN_DROPSQL('DROP SEQUENCE lab_result_cm_seq'); +END; +/ +create sequence lab_result_cm_seq +/ + +create or replace trigger lab_result_cm_trg +before insert on lab_result_cm +for each row +begin + select lab_result_cm_seq.nextval into :new.LAB_RESULT_CM_ID from dual; +end; +/ + +BEGIN +PMN_DROPSQL('DROP TABLE priority'); +END; +/ + +CREATE TABLE PRIORITY ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + PRIORITY VARCHAR2(50) NULL + ) +/ + +BEGIN +PMN_DROPSQL('DROP TABLE location'); +END; +/ + +CREATE TABLE LOCATION ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + RESULT_LOC VARCHAR2(50) NULL + ) +/ +create or replace procedure PCORNetLabResultCM as +begin + +PMN_DROPSQL('drop index lab_result_cm_idx'); +PMN_DROPSQL('drop index priority_idx'); +PMN_DROPSQL('drop index location_idx'); + +execute immediate 'truncate table priority'; +execute immediate 'truncate table location'; +execute immediate 'truncate table lab_result_cm'; + +insert into priority +select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode PRIORITY +from i2b2fact +inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num +inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode +where c_fullname LIKE '\PCORI_MOD\PRIORITY\%'; + +execute immediate 'create index priority_idx on priority (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('PRIORITY'); + +insert into location +select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode RESULT_LOC +from i2b2fact +inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num +inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode +where c_fullname LIKE '\PCORI_MOD\RESULT_LOC\%'; + +execute immediate 'create index location_idx on location (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('LOCATION'); + +INSERT INTO lab_result_cm + (PATID + ,ENCOUNTERID + ,LAB_NAME + ,SPECIMEN_SOURCE + ,LAB_LOINC + ,PRIORITY + ,RESULT_LOC + ,LAB_PX + ,LAB_PX_TYPE + ,LAB_ORDER_DATE + ,SPECIMEN_DATE + ,SPECIMEN_TIME + ,RESULT_DATE + ,RESULT_TIME + ,RESULT_QUAL + ,RESULT_NUM + ,RESULT_MODIFIER + ,RESULT_UNIT + ,NORM_RANGE_LOW + ,NORM_MODIFIER_LOW + ,NORM_RANGE_HIGH + ,NORM_MODIFIER_HIGH + ,ABN_IND + ,RAW_LAB_NAME + ,RAW_LAB_CODE + ,RAW_PANEL + ,RAW_RESULT + ,RAW_UNIT + ,RAW_ORDER_DEPT + ,RAW_FACILITY_CODE) + +SELECT DISTINCT M.patient_num patid, +M.encounter_num encounterid, +CASE WHEN ont_parent.C_BASECODE LIKE 'LAB_NAME%' then SUBSTR (ont_parent.c_basecode,10, 10) ELSE 'UN' END LAB_NAME, +CASE WHEN lab.pcori_specimen_source like '%or SR_PLS' THEN 'SR_PLS' WHEN lab.pcori_specimen_source is null then 'NI' ELSE lab.pcori_specimen_source END specimen_source, -- (Better way would be to fix the column in the ontology but this will work) +NVL(lab.pcori_basecode, 'NI') LAB_LOINC, +NVL(p.PRIORITY,'NI') PRIORITY, +NVL(l.RESULT_LOC,'NI') RESULT_LOC, +NVL(lab.pcori_basecode, 'NI') LAB_PX, +'LC' LAB_PX_TYPE, +m.start_date LAB_ORDER_DATE, +m.start_date SPECIMEN_DATE, +to_char(m.start_date,'HH24:MI') SPECIMEN_TIME, +m.end_date RESULT_DATE, +to_char(m.end_date,'HH24:MI') RESULT_TIME, +--CASE WHEN m.ValType_Cd='T' THEN NVL(nullif(m.TVal_Char,''),'NI') ELSE 'NI' END RESULT_QUAL, -- TODO: Should be a standardized value +'NI' RESULT_QUAL, -- Local fix for KUMC (temp) +CASE WHEN m.ValType_Cd='N' THEN m.NVAL_NUM ELSE null END RESULT_NUM, +CASE WHEN m.ValType_Cd='N' THEN (CASE NVL(nullif(m.TVal_Char,''),'NI') WHEN 'E' THEN 'EQ' WHEN 'NE' THEN 'OT' WHEN 'L' THEN 'LT' WHEN 'LE' THEN 'LE' WHEN 'G' THEN 'GT' WHEN 'GE' THEN 'GE' ELSE 'NI' END) ELSE 'TX' END RESULT_MODIFIER, +--NVL(m.Units_CD,'NI') RESULT_UNIT, -- TODO: Should be standardized units +CASE + WHEN INSTR(m.Units_CD, '%') > 0 THEN 'PERCENT' + WHEN m.Units_CD IS NULL THEN NVL(m.Units_CD,'NI') + when length(m.Units_CD) > 11 then substr(m.Units_CD, 1, 11) + ELSE TRIM(REPLACE(UPPER(m.Units_CD), '(CALC)', '')) +end RESULT_UNIT, -- Local fix for KUMC +norm.ref_lo NORM_RANGE_LOW, +case + when norm.ref_lo is not null and norm.ref_hi is not null then 'EQ' + when norm.ref_lo is not null and norm.ref_hi is null then 'GE' + when norm.ref_lo is null and norm.ref_hi is not null then 'NO' +else 'NI' +end NORM_MODIFIER_LOW, +norm.ref_hi NORM_RANGE_HIGH, +case + when norm.ref_lo is not null and norm.ref_hi is not null then 'EQ' + when norm.ref_lo is not null and norm.ref_hi is null then 'NO' + when norm.ref_lo is null and norm.ref_hi is not null then 'LE' + else 'NI' +end NORM_MODIFIER_HIGH, +CASE NVL(nullif(m.VALUEFLAG_CD,''),'NI') WHEN 'H' THEN 'AH' WHEN 'L' THEN 'AL' WHEN 'A' THEN 'AB' ELSE 'NI' END ABN_IND, +NULL RAW_LAB_NAME, +NULL RAW_LAB_CODE, +NULL RAW_PANEL, +--CASE WHEN m.ValType_Cd='T' THEN m.TVal_Char ELSE to_char(m.NVal_Num) END RAW_RESULT, +CASE WHEN m.ValType_Cd='T' THEN substr(m.TVal_Char, 1, 50) ELSE to_char(m.NVal_Num) END RAW_RESULT, -- Local fix for KUMC +NULL RAW_UNIT, +NULL RAW_ORDER_DEPT, +m.concept_cd RAW_FACILITY_CODE + +FROM i2b2fact M +inner join encounter enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num -- Constraint to selected encounters +inner join demographic demo on demo.patid=m.patient_num +inner join pcornet_lab lab on lab.c_basecode = M.concept_cd and lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' +inner JOIN pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname + +LEFT OUTER JOIN priority p + +ON M.patient_num=p.patient_num +and M.encounter_num=p.encounter_num +and M.provider_id=p.provider_id +and M.concept_cd=p.concept_Cd +and M.start_date=p.start_Date + +LEFT OUTER JOIN location l + +ON M.patient_num=l.patient_num +and M.encounter_num=l.encounter_num +and M.provider_id=l.provider_id +and M.concept_cd=l.concept_Cd +and M.start_date=l.start_Date + +LEFT OUTER JOIN labnormal norm + on m.concept_cd=norm.concept_cd + and demo.sex=norm.sex + and (m.start_date - demo.birth_date) > norm.age_lower + and (m.start_date - demo.birth_date) <= norm.age_upper + +WHERE m.ValType_Cd in ('N','T') +and m.MODIFIER_CD='@' +and (m.nval_num is null or m.nval_num<=9999999) -- exclude lengths that exceed the spec +; + +execute immediate 'create index lab_result_cm_idx on lab_result_cm (PATID, ENCOUNTERID)'; +--GATHER_TABLE_STATS('LAB_RESULT_CM'); + +END PCORNetLabResultCM; +/ +BEGIN +PCORNetLabResultCM(); +END; +/ +SELECT count(LAB_RESULT_CM_ID) from lab_result_cm where rownum = 1 +--SELECT 1 FROM dual \ No newline at end of file diff --git a/Oracle/med_admin_init.sql b/Oracle/med_admin_init.sql index 6750d07..c2eea9c 100644 --- a/Oracle/med_admin_init.sql +++ b/Oracle/med_admin_init.sql @@ -1,2 +1,2 @@ -/*Verify that there is useful data in the observation_fact table. If not, check for failed Heron ETL.*/ +/*Verify that there is useful data in the observation_fact table. If not, Heron ETL may have failed.*/ SELECT count(patient_num) from BLUEHERONDATA.observation_fact where modifier_cd = 'MedObs|MAR:Given' and rownum = 1; \ No newline at end of file diff --git a/Oracle/obs_clin.sql b/Oracle/obs_clin.sql new file mode 100644 index 0000000..813e118 --- /dev/null +++ b/Oracle/obs_clin.sql @@ -0,0 +1,31 @@ +-------------------------------------------------------------------------------- +-- OBS_CLIN +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE obs_clin'); +END; +/ +CREATE TABLE obs_clin( + OBSCLINID varchar(50) NOT NULL, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, + OBSCLIN_PROVIDERID varchar(50) NULL, + OBSCLIN_DATE date NULL, + OBSCLIN_TIME varchar(5) NULL, + OBSCLIN_TYPE varchar(2) NULL, + OBSCLIN_CODE varchar(50) NULL, + OBSCLIN_RESULT_QUAL varchar(50) NULL, + OBSCLIN_RESULT_TEXT varchar(50) NULL, + OBSCLIN_RESULT_SNOMED varchar(50), + OBSCLIN_RESULT_NUM NUMBER(18, 0) NULL, -- (8,0) + OBSCLIN_RESULT_MODIFIER varchar(2) NULL, + OBSCLIN_RESULT_UNIT varchar(50) NULL, + RAW_OBSCLIN_NAME varchar(50) NULL, + RAW_OBSCLIN_CODE varchar(50) NULL, + RAW_OBSCLIN_TYPE varchar(50) NULL, + RAW_OBSCLIN_RESULT varchar(50) NULL, + RAW_OBSCLIN_MODIFIER varchar(50) NULL, + RAW_OBSCLIN_UNIT varchar(50) NULL +) +/ \ No newline at end of file diff --git a/Oracle/obs_gen.sql b/Oracle/obs_gen.sql new file mode 100644 index 0000000..e1c83fc --- /dev/null +++ b/Oracle/obs_gen.sql @@ -0,0 +1,31 @@ +-------------------------------------------------------------------------------- +-- OBS_GEN +-------------------------------------------------------------------------------- + +BEGIN +PMN_DROPSQL('DROP TABLE obs_gen'); +END; +/ +CREATE TABLE obs_gen( + OBSGENID varchar(50) NOT NULL, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, + OBSGEN_PROVIDERID varchar(50) NULL, + OBSGEN_DATE date NULL, + OBSGEN_TIME varchar(5) NULL, + OBSGEN_TYPE varchar(30) NULL, + OBSGEN_CODE varchar(50) NULL, + OBSGEN_RESULT_QUAL varchar(50) NULL, + OBSGEN_RESULT_TEXT varchar(50) NULL, + OBSGEN_RESULT_NUM NUMBER(18, 0) NULL, -- (8,0) + OBSGEN_RESULT_MODIFIER varchar(2) NULL, + OBSGEN_RESULT_UNIT varchar(50) NULL, + OBSGEN_TABLE_MODIFIED varchar(3) NULL, + OBSGEN_ID_MODIFIED varchar(50) NULL, + RAW_OBSGEN_NAME varchar(50) NULL, + RAW_OBSGEN_CODE varchar(50) NULL, + RAW_OBSGEN_TYPE varchar(50) NULL, + RAW_OBSGEN_RESULT varchar(50) NULL, + RAW_OBSGEN_UNIT varchar(50) NULL +) +/ \ No newline at end of file diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql new file mode 100644 index 0000000..1ab9c38 --- /dev/null +++ b/Oracle/pcornet_init.sql @@ -0,0 +1,219 @@ +-------------------------------------------------------------------------------- +-- HELPER FUNCTIONS AND PROCEDURES +-------------------------------------------------------------------------------- + +create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS + BEGIN + DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. + tabname => table_name, + estimate_percent => 50, -- Percentage picked somewhat arbitrarily + cascade => TRUE, + degree => 16 + ); +END GATHER_TABLE_STATS; +/ + + +create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS + BEGIN + EXECUTE IMMEDIATE sqlstring; + EXCEPTION + WHEN OTHERS THEN NULL; +END PMN_DROPSQL; +/ + + +create or replace FUNCTION PMN_IFEXISTS(objnamestr VARCHAR2, objtypestr VARCHAR2) RETURN BOOLEAN AS +cnt NUMBER; +BEGIN + SELECT COUNT(*) + INTO cnt + FROM USER_OBJECTS + WHERE upper(OBJECT_NAME) = upper(objnamestr) + and upper(object_type) = upper(objtypestr); + + IF( cnt = 0 ) + THEN + --dbms_output.put_line('NO!'); + return FALSE; + ELSE + --dbms_output.put_line('YES!'); + return TRUE; + END IF; + +END PMN_IFEXISTS; +/ + + +create or replace PROCEDURE PMN_Execuatesql(sqlstring VARCHAR2) AS +BEGIN + EXECUTE IMMEDIATE sqlstring; + dbms_output.put_line(sqlstring); +END PMN_ExecuateSQL; +/ + + +--ACK: http://dba.stackexchange.com/questions/9441/how-to-catch-and-handle-only-specific-oracle-exceptions +create or replace procedure create_error_table(table_name varchar2) as +sqltext varchar2(4000); + +begin + dbms_errlog.create_error_log(dml_table_name => table_name); +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -955 THEN + NULL; -- suppresses ORA-00955 exception ("name is already used by an existing object") + ELSE + RAISE; + END IF; +-- Delete rows from a previous run in case the table already existed +sqltext := 'delete from ERR$_' || table_name; +PMN_Execuatesql(sqltext); +end; +/ + + +create or replace FUNCTION GETDATAMARTID RETURN VARCHAR2 IS +BEGIN + RETURN '&&datamart_id'; +END; +/ + + +CREATE OR REPLACE FUNCTION GETDATAMARTNAME RETURN VARCHAR2 AS +BEGIN + RETURN '&&datamart_name'; +END; +/ + + +CREATE OR REPLACE FUNCTION GETDATAMARTPLATFORM RETURN VARCHAR2 AS +BEGIN + RETURN '02'; -- 01 is MSSQL, 02 is Oracle +END; +/ + + +BEGIN +PMN_DROPSQL('DROP TABLE pcornet_codelist'); +END; +/ +create table pcornet_codelist(codetype varchar2(20), code varchar2(50)) +/ + +create or replace procedure pcornet_parsecode (codetype in varchar, codestring in varchar) as + +tex varchar(2000); +pos number(9); +readstate char(1) ; +nextchar char(1) ; +val varchar(50); + +begin + +val:=''; +readstate:='F'; +pos:=0; +tex := codestring; +FOR pos IN 1..length(tex) +LOOP +-- dbms_output.put_line(val); + nextchar:=substr(tex,pos,1); + if nextchar!=',' then + if nextchar='''' then + if readstate='F' then + val:=''; + readstate:='T'; + else + insert into pcornet_codelist values (codetype,val); + val:=''; + readstate:='F' ; + end if; + else + if readstate='T' then + val:= val || nextchar; + end if; + end if; + end if; +END LOOP; + +end pcornet_parsecode; +/ + +create or replace procedure pcornet_popcodelist as + +codedata varchar(2000); +onecode varchar(20); +codetype varchar(20); + +cursor getcodesql is +select 'RACE',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' +union +select 'SEX',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' +union +select 'HISPANIC',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%'; + +begin +open getcodesql; +LOOP + fetch getcodesql into codetype,codedata; + EXIT WHEN getcodesql%NOTFOUND ; + pcornet_parsecode (codetype,codedata ); +end loop; + +close getcodesql ; +end pcornet_popcodelist; +/ + + +-------------------------------------------------------------------------------- +-- I2B2 SYNONYMS, VIEWS, AND INTERMEDIARY TABLES +-------------------------------------------------------------------------------- + + +CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT +/ + +CREATE OR REPLACE SYNONYM I2B2MEDFACT FOR OBSERVATION_FACT_MEDS +/ + +BEGIN +PMN_DROPSQL('DROP TABLE i2b2patient_list'); +END; +/ + +CREATE table i2b2patient_list as +select * from +( +select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') +) where ROWNUM<100000000 +/ + +create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) +/ + +create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE >> variables = dict(I2B2STAR='I2B2DEMODATA') >>> txform = SqlScriptTask( ... account='sqlite:///', passkey=None, - ... script=Script.migrate_fact_upload, + ... script=Script.pcornet_loader, ... param_vars=variables) ISSUE: doctest dependencies? @@ -856,7 +856,7 @@ class MigrateUpload(SqlScriptTask, I2B2Task): parallel_degree = IntParam(default=24, significant=False) - script = Script.migrate_fact_upload + script = Script.pcornet_loader @property def variables(self) -> Environment: diff --git a/i2p_tasks.py b/i2p_tasks.py index 2d10e1e..6a197c6 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -4,41 +4,145 @@ from sql_syntax import Environment from typing import List +class Condition(SqlScriptTask): + script = Script.condition -class PCORNetInit(SqlScriptTask): - script = Script.PCORNetInit + def requires(self) -> List[luigi.Task]: + return [Encounter()] + +class Death(SqlScriptTask): + script = Script.death + + def requires(self) -> List[luigi.Task]: + return [Demographic()] + +class DeathCause(SqlScriptTask): + script = Script.death_cause + +class Demographic(SqlScriptTask): + script = Script.demographic + + def requires(self) -> List[luigi.Task]: + return [PcornetInit()] + +class Diagnosis(SqlScriptTask): + script = Script.diagnosis + + def requires(self) -> List[luigi.Task]: + return [Encounter()] @property def variables(self) -> Environment: - return dict(datamart_id='todo1', datamart_name='todo2', i2b2_data_schema='BLUEHERONDATA', min_pat_list_date_dd_mon_rrrr='20-JAN-18', min_visit_date_dd_mon_rrrr='20-JAN-18', - i2b2_meta_schema='BLUEHERONMETADATA') + return dict(i2b2_meta_schema='BLUEHERONMETADATA') + +class Dispensing(SqlScriptTask): + script = Script.dispensing -class PCORNetLoader_ora(SqlScriptTask): - script = Script.PCORNetLoader_ora + def requires(self) -> List[luigi.Task]: + return [Encounter()] @property def variables(self) -> Environment: - return dict(i2b2_meta_schema='BLUEHERONMETADATA', network_id='todo1', network_name='todo2', enrollment_months_back='2') + return dict(i2b2_meta_schema='BLUEHERONMETADATA') + + +class Encounter(SqlScriptTask): + script = Script.encounter def requires(self) -> List[luigi.Task]: - '''Wrap each of `self.script.deps()` in a SqlScriptTask. - ''' - return [PCORNetInit()] + return [Demographic()] -class PCORNetMed_Admin(SqlScriptTask): - script = Script.med_admin +class Enrollment(SqlScriptTask): + script = Script.enrollment + + @property + def variables(self) -> Environment: + return dict(enrollment_months_back='2') + +class Harvest(SqlScriptTask): + script = Script.harvest + + def requires(self) -> List[luigi.Task]: + return [Condition(), Death(), Diagnosis(), Dispensing(), LabResultCM(), Prescribing(), Procedures(), Vital()] @property def variables(self) -> Environment: - return dict() + return dict(network_id='C4', network_name='GPC') + +class LabResultCM(SqlScriptTask): + script = Script.lab_result_cm def requires(self) -> List[luigi.Task]: + return [Encounter()] - return [PCORNetMed_Admin_Init()] +class MedAdmin(SqlScriptTask): + script = Script.med_admin -class PCORNetMed_Admin_Init(SqlScriptTask): + def requires(self) -> List[luigi.Task]: + return [MedAdminInit()] + +class MedAdminInit(SqlScriptTask): script = Script.med_admin_init +class ObsClin(SqlScriptTask): + script = Script.obs_clin + + def requires(self) -> List[luigi.Task]: + return [] + +class ObsGen(SqlScriptTask): + script = Script.obs_gen + + def requires(self) -> List[luigi.Task]: + return [] + +class PcornetInit(SqlScriptTask): + script = Script.pcornet_init + @property def variables(self) -> Environment: - return dict() \ No newline at end of file + return dict(datamart_id='C4UK', datamart_name='University of Kansas', i2b2_data_schema='BLUEHERONDATA', min_pat_list_date_dd_mon_rrrr='01-Jan-2010', min_visit_date_dd_mon_rrrr='01-Jan-2010', + i2b2_meta_schema='BLUEHERONMETADATA') + +class PcornetLoader(SqlScriptTask): + script = Script.pcornet_loader + + def requires(self) -> List[luigi.Task]: + #return [DeathCause(), Enrollment(), Harvest(), MedAdmin(), PcornetTrail(), ProCM()] + return [DeathCause(), Demographic(), MedAdmin()] + +class PcornetTrail(SqlScriptTask): + script = Script.pcornet_trail + + def requires(self) -> List[luigi.Task]: + return [] + +class Prescribing(SqlScriptTask): + script = Script.prescribing + + def requires(self) -> List[luigi.Task]: + return [Encounter()] + +class ProCM(SqlScriptTask): + script = Script.pro_cm + + def requires(self) -> List[luigi.Task]: + return [] + +class Procedures(SqlScriptTask): + script = Script.procedures + + def requires(self) -> List[luigi.Task]: + return [Encounter()] + +class Provider(SqlScriptTask): + script = Script.provider + + def requires(self) -> List[luigi.Task]: + return [] + +class Vital(SqlScriptTask): + script = Script.vital + + def requires(self) -> List[luigi.Task]: + return [Encounter()] \ No newline at end of file diff --git a/script_lib.py b/script_lib.py index 92147b5..70f6482 100644 --- a/script_lib.py +++ b/script_lib.py @@ -265,26 +265,54 @@ class Script(ScriptMixin, enum.Enum): ''' [ # Keep sorted - PCORNetInit, - PCORNetLoader_ora, - epic_facts_load, - epic_flowsheets_transform, - etl_tests_init, + condition, + death, + death_cause, + demographic, + diagnosis, + dispensing, + encounter, + enrollment, + harvest, + lab_result_cm, med_admin, med_admin_init, - migrate_fact_upload, + obs_clin, + obs_gen, + pcornet_init, + pcornet_loader, + pcornet_trail, + prescribing, + pro_cm, + procedures, + provider, + vital ] = [ pkg.resource_string(__name__, 'Oracle/' + fname).decode('utf-8') for fname in [ - 'PCORNetInit.sql', - 'PCORNetLoader_ora.sql', - 'epic_facts_load.sql', - 'epic_flowsheets_transform.sql', - 'etl_tests_init.sql', + 'condition.sql', + 'death.sql', + 'death_cause.sql', + 'demographic.sql', + 'diagnosis.sql', + 'dispensing.sql', + 'encounter.sql', + 'enrollment.sql', + 'harvest.sql', + 'lab_result_cm.sql', 'med_admin.sql', 'med_admin_init.sql', - 'migrate_fact_upload.sql', + 'obs_clin.sql', + 'obs_gen.sql', + 'pcornet_init.sql', + 'pcornet_loader.sql', + 'pcornet_trail.sql', + 'prescribing.sql', + 'pro_cm.sql', + 'procedures.sql', + 'provider.sql', + 'vital.sql' ] ] From fd31b9a23e1688c6516d1cec789f95672fc6de58 Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 19 Feb 2018 16:40:53 -0600 Subject: [PATCH 325/507] Restored Init and Loader sql to original state. --- Oracle/PCORNetInit.sql | 87 +----------------------------------- Oracle/PCORNetLoader_ora.sql | 4 +- 2 files changed, 2 insertions(+), 89 deletions(-) diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql index 88f6f10..2cfa68c 100644 --- a/Oracle/PCORNetInit.sql +++ b/Oracle/PCORNetInit.sql @@ -1006,89 +1006,4 @@ CREATE TABLE harvest( REFRESH_DEATH_DATE date NULL, REFRESH_DEATH_CAUSE_DATE date NULL ) -/ - --------------------------------------------------------------------------------- --- OBS_CLIN --------------------------------------------------------------------------------- - -BEGIN -PMN_DROPSQL('DROP TABLE obs_clin'); -END; -/ -CREATE TABLE obs_clin( - OBSCLINID varchar(50) NOT NULL, - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NULL, - OBSCLIN_PROVIDERID varchar(50) NULL, - OBSCLIN_DATE date NULL, - OBSCLIN_TIME varchar(5) NULL, - OBSCLIN_TYPE varchar(2) NULL, - OBSCLIN_CODE varchar(50) NULL, - OBSCLIN_RESULT_QUAL varchar(50) NULL, - OBSCLIN_RESULT_TEXT varchar(50) NULL, - OBSCLIN_RESULT_SNOMED varchar(50), - OBSCLIN_RESULT_NUM NUMBER(18, 0) NULL, -- (8,0) - OBSCLIN_RESULT_MODIFIER varchar(2) NULL, - OBSCLIN_RESULT_UNIT varchar(50) NULL, - RAW_OBSCLIN_NAME varchar(50) NULL, - RAW_OBSCLIN_CODE varchar(50) NULL, - RAW_OBSCLIN_TYPE varchar(50) NULL, - RAW_OBSCLIN_RESULT varchar(50) NULL, - RAW_OBSCLIN_MODIFIER varchar(50) NULL, - RAW_OBSCLIN_UNIT varchar(50) NULL -) -/ - --------------------------------------------------------------------------------- --- OBS_GEN --------------------------------------------------------------------------------- - -BEGIN -PMN_DROPSQL('DROP TABLE obs_gen'); -END; -/ -CREATE TABLE obs_gen( - OBSGENID varchar(50) NOT NULL, - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NULL, - OBSGEN_PROVIDERID varchar(50) NULL, - OBSGEN_DATE date NULL, - OBSGEN_TIME varchar(5) NULL, - OBSGEN_TYPE varchar(30) NULL, - OBSGEN_CODE varchar(50) NULL, - OBSGEN_RESULT_QUAL varchar(50) NULL, - OBSGEN_RESULT_TEXT varchar(50) NULL, - OBSGEN_RESULT_NUM NUMBER(18, 0) NULL, -- (8,0) - OBSGEN_RESULT_MODIFIER varchar(2) NULL, - OBSGEN_RESULT_UNIT varchar(50) NULL, - OBSGEN_TABLE_MODIFIED varchar(3) NULL, - OBSGEN_ID_MODIFIED varchar(50) NULL, - RAW_OBSGEN_NAME varchar(50) NULL, - RAW_OBSGEN_CODE varchar(50) NULL, - RAW_OBSGEN_TYPE varchar(50) NULL, - RAW_OBSGEN_RESULT varchar(50) NULL, - RAW_OBSGEN_UNIT varchar(50) NULL -) -/ - --------------------------------------------------------------------------------- --- PROVIDER --------------------------------------------------------------------------------- - -BEGIN -PMN_DROPSQL('DROP TABLE provider'); -END; -/ -CREATE TABLE provider( - PROVIDERID varchar(50) NOT NULL, - PROVIDER_SEX varchar(2) NULL, - PROVIDER_SPECIALTY_PRIMARY varchar(50) NULL, - PROVIDER_NPI NUMBER(18, 0) NULL, -- (8,0) - PROVIDER_NPI_FLAG varchar(1) NULL, - RAW_PROVIDER_SPECIALTY_PRIMARY varchar(50) NULL -) -/ - - -SELECT 0 FROM dual +/ \ No newline at end of file diff --git a/Oracle/PCORNetLoader_ora.sql b/Oracle/PCORNetLoader_ora.sql index 8a28e78..a2ecd11 100644 --- a/Oracle/PCORNetLoader_ora.sql +++ b/Oracle/PCORNetLoader_ora.sql @@ -1288,6 +1288,4 @@ begin PCORNetPostProc; end PCORNetLoader; -/ - -SELECT 1 FROM HARVEST \ No newline at end of file +/ \ No newline at end of file From ff872f854539244057f0dfa0743743fc331bc32c Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 21 Feb 2018 10:02:24 -0600 Subject: [PATCH 326/507] cite GPC --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 1b534c8..26ee328 100644 --- a/README.md +++ b/README.md @@ -56,3 +56,10 @@ For design info and project coding style, see [CONTRIBUTING][]. [CONTRIBUTING]: CONTRIBUTING.md +## References + + - Waitman, L.R., Aaronson, L.S., Nadkarni, P.M., Connolly, D.W. & + Campbell, J.R. [The Greater Plains Collaborative: a PCORnet Clinical + Research Data Network][1]. J Am Med Inform Assoc 21, 637-641 (2014). + +[1]: https://www.ncbi.nlm.nih.gov/pubmed/24778202 From 3b38c5af3683d8d052eaa85dcfc57b933cb58fe2 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 21 Feb 2018 10:19:49 -0600 Subject: [PATCH 327/507] restore KUMC copyright --- LICENSE | 22 ++++++++++++++++++++++ README.md | 8 ++++++++ 2 files changed, 30 insertions(+) diff --git a/LICENSE b/LICENSE index 61a4be2..c4cf5f0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,25 @@ +Copyright (c) 2014-2018 Univeristy of Kansas Medical Center + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + i2b2 Software License (“Software Licenseâ€) Version 2.1 diff --git a/README.md b/README.md index 26ee328..590f27d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,14 @@ # i2p-transform i2b2 to PCORnet Common Data Model Transformation - requires i2b2 PCORnet ontology +## Copyright and License + +Copyright (c) 2014-2017 Univeristy of Kansas Medical Center +License: MIT + +Portions copyright The Brigham and Women’s Hospital, Inc. +License: i2b2 Software License + # HERON ETL to i2b2 We (*@@TODO aim to*) transform the data from a number of sources and From e92dd6909dc542c6230edfcf34b74f08a8c7aa05 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 21 Feb 2018 11:13:45 -0600 Subject: [PATCH 328/507] README: put i2p-transform in context - prune HERON stuff --- README.md | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 590f27d..6deca2c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,19 @@ # i2p-transform -i2b2 to PCORnet Common Data Model Transformation - requires i2b2 PCORnet ontology + +The Greater Plains Collaborative ([GPC][]) and the Accessible Research +Commons for Health ([ARCH][]) are PCORI CDRNs that i2b2 as a core +technology. The [i2b2 PCORnet Common Data Model Ontology][ont] is a +representation of the PCORnet Common Data Model ([CDM][]) which allows +not only i2b2 style ad-hoc query of clinical data but also this +i2p-transform of data from an [i2b2 datamart][CRC] into a PCORNet CDM +datamart. + +[GPC]: http://pcornet.org/clinical-data-research-networks/cdrn4-university-of-kansas-medical-center-great-plains-collaborative/ +[ARCH]: http://pcornet.org/clinical-data-research-networks/cdrn1-harvard-university-scihls/ +[ont]: https://github.com/ARCH-commons/arch-ontology +[CDM]: http://www.pcornet.org/pcornet-common-data-model/ +[CRC]: https://www.i2b2.org/software/files/PDF/current/CRC_Design.pdf + ## Copyright and License @@ -9,13 +23,6 @@ License: MIT Portions copyright The Brigham and Women’s Hospital, Inc. License: i2b2 Software License -# HERON ETL to i2b2 - -We (*@@TODO aim to*) transform the data from a number of sources and -load it into an i2b2 data repository, following the -[i2b2 Data Repository (CRC) Cell Design Document][CRC]. - -[CRC]: https://www.i2b2.org/software/files/PDF/current/CRC_Design.pdf ## Usage From b12d8863a612f268de8740f128b1cab1fc6bc9c4 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 21 Feb 2018 11:27:39 -0600 Subject: [PATCH 329/507] README: update basic usage --- README.md | 14 +++----------- requirements.txt | 8 ++++---- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6deca2c..a70b6fb 100644 --- a/README.md +++ b/README.md @@ -30,21 +30,13 @@ To build flowsheet transformation views, once installation and configuration (below) is done, invoke the `FlowsheetViews` task following [luigi][] conventions: - luigi --module epic_flowsheets FlowsheetViews + luigi --module i2p_tasks MedAdmin In more detail: - PYTHONPATH=. LUIGI_CONFIG_PATH=heron-test.cfg luigi \ + PYTHONPATH=. LUIGI_CONFIG_PATH=my-client.cfg luigi \ --local-scheduler \ - --module epic_flowsheets FlowsheetViews - -*@@TODO: actual fact loading* - -Then use `etl_tasks.MigratePendingUploads` to install the data in the -production i2b2 `observation_fact` table. Use -`@@TODO.PatientDimension` and `@@TODO.VisitDimLoad` to load the -`patient_dimension` and `visit_dimension` tables from -`observation_fact`. + --module i2p_tasks MedAdmin [luigi]: https://github.com/spotify/luigi diff --git a/requirements.txt b/requirements.txt index 5a42179..2a5e594 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,13 +3,13 @@ # Usage (development): # -# $ virtualenv --python python3.5 ~/pyenv/grouse3 -# $ . ~/pyenv/grouse3/bin/activate -# (grouse3)$ pip install -r requirements.txt +# $ virtualenv --python python3 ~/pyenv/luigi_cdm +# $ . ~/pyenv/luigi_cdm/bin/activate +# (luigi_cdm)$ pip install -r requirements.txt # ... # Successfully built luigi tornado # Installing collected packages: ... -# ... tornado-4.4.2 +# ... tornado-4.5.3 # Production usage is more like: From fb605c9e33ca2117eb359b5ce752237c7b0b4dda Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 21 Feb 2018 11:48:08 -0600 Subject: [PATCH 330/507] script_lib: title, statements tests for med_admin --- Oracle/med_admin.sql | 5 +++++ script_lib.py | 15 ++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index f029de3..93557e8 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -1,3 +1,8 @@ +/** med_admin - Build Medication Administration table. + +Copyright (c) 2018 University of Kansas Medical Center +*/ + -------------------------------------------------------------------------------- -- HELPER FUNCTIONS AND PROCEDURES -------------------------------------------------------------------------------- diff --git a/script_lib.py b/script_lib.py index 70f6482..16646cd 100644 --- a/script_lib.py +++ b/script_lib.py @@ -4,22 +4,23 @@ Each script should have a title, taken from the first line:: - >>> Script.migrate_fact_upload.title - 'append data from a workspace table.' + >>> Script.med_admin.title + 'Build Medication Administration table.' - >>> text = Script.migrate_fact_upload.value + >>> text = Script.med_admin.value >>> lines = text.split('\n') >>> print(lines[0]) ... #doctest: +NORMALIZE_WHITESPACE - /** migrate_fact_upload - append data from a workspace table. + /** med_admin - Build Medication Administration table. We can separate the script into statements:: - >>> statements = Script.epic_flowsheets_transform.statements() + >>> statements = Script.med_admin.statements() >>> print(next(s for s in statements if 'insert' in s)) ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE - insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) - with test_key as ( ... + create or replace trigger med_admin_trg + before insert on med_admin + ... A bit of sqlplus syntax is supported for ignoring errors in just part of a script: From e3f4f7505b7b69a7b1806327c5d3554a92808eaf Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 22 Feb 2018 10:52:29 -0600 Subject: [PATCH 331/507] Increased size of raw name and route fields. --- Oracle/med_admin.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index f029de3..889ca38 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -41,15 +41,15 @@ CREATE TABLE med_admin( MEDADMIN_STOP_TIME varchar(5) NULL, MEDADMIN_TYPE varchar(2) NULL, MEDADMIN_CODE varchar(50) NULL, - MEDADMIN_DOSE_ADMIN NUMBER(18, 2) NULL, -- (8,0) + MEDADMIN_DOSE_ADMIN NUMBER(18, 5) NULL, -- (8,0) MEDADMIN_DOSE_ADMIN_UNIT varchar(50) NULL, MEDADMIN_ROUTE varchar(50) NULL, MEDADMIN_SOURCE varchar(2) NULL, - RAW_MEDADMIN_MED_NAME varchar(50) NULL, + RAW_MEDADMIN_MED_NAME varchar(2000) NULL, RAW_MEDADMIN_CODE varchar(50) NULL, RAW_MEDADMIN_DOSE_ADMIN varchar(50) NULL, RAW_MEDADMIN_DOSE_ADMIN_UNIT varchar(50) NULL, - RAW_MEDADMIN_ROUTE varchar(50) NULL + RAW_MEDADMIN_ROUTE varchar(100) NULL ) / BEGIN From 7fc8ea47023e7512f1f1673e3cd7095c0c685fcc Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 1 Mar 2018 13:26:54 -0600 Subject: [PATCH 332/507] Dynamically typed SqlScriptTask dependencies for visualization --- etl_tasks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/etl_tasks.py b/etl_tasks.py index 9ff69b8..14948ca 100644 --- a/etl_tasks.py +++ b/etl_tasks.py @@ -233,11 +233,11 @@ def vars_for_deps(self) -> Environment: def requires(self) -> List[luigi.Task]: '''Wrap each of `self.script.deps()` in a SqlScriptTask. ''' - return [SqlScriptTask(script=s, - param_vars=self.vars_for_deps, - account=self.account, - passkey=self.passkey, - echo=self.echo) + return [type(s.name, (SqlScriptTask,), {'script':s, + 'param_vars':self.vars_for_deps, + 'account':self.account, + 'passkey':self.passkey, + 'echo':self.echo})() for s in self.script.deps()] def log_info(self) -> Dict[str, Any]: From af01510a20849e69c6accbef6ac2b198ecb83dc2 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 2 Mar 2018 14:39:45 -0600 Subject: [PATCH 333/507] Luigi CDM full build with table create but not populate --- Oracle/condition.sql | 18 +- Oracle/death.sql | 16 +- Oracle/death_cause.sql | 14 +- Oracle/demographic.sql | 236 +++++++++++------- Oracle/diagnosis.sql | 18 +- Oracle/dispensing.sql | 18 +- Oracle/encounter.sql | 19 +- Oracle/enrollment.sql | 21 +- Oracle/harvest.sql | 32 ++- Oracle/lab_result_cm.sql | 18 +- Oracle/med_admin.sql | 40 +-- Oracle/pcornet_init.sql | 60 +---- Oracle/pcornet_loader.sql | 16 +- .../{pcornet_trail.sql => pcornet_trial.sql} | 12 +- Oracle/prescribing.sql | 18 +- Oracle/pro_cm.sql | 12 +- Oracle/procedures.sql | 15 +- Oracle/vital.sql | 17 +- i2p_tasks.py | 121 +++------ script_lib.py | 4 +- 20 files changed, 361 insertions(+), 364 deletions(-) rename Oracle/{pcornet_trail.sql => pcornet_trial.sql} (56%) diff --git a/Oracle/condition.sql b/Oracle/condition.sql index bfe257e..cedd16f 100644 --- a/Oracle/condition.sql +++ b/Oracle/condition.sql @@ -1,7 +1,8 @@ --------------------------------------------------------------------------------- --- CONDITION --------------------------------------------------------------------------------- +/** condition - create and populate the condition table. +*/ +select encounterid from encounter where 'dep' = 'encounter.sql' +/ BEGIN PMN_DROPSQL('DROP TABLE condition'); END; @@ -100,9 +101,10 @@ execute immediate 'create index condition_idx on condition (PATID, ENCOUNTERID)' end PCORNetCondition; / -BEGIN -PCORNetCondition(); -END; +insert into cdm_status (status, last_update) values ('condition', sysdate) +--BEGIN +--PCORNetCondition(); +--END; / -SELECT count(CONDITIONID) from condition where rownum = 1 ---SELECT 1 FROM dual \ No newline at end of file +select 1 from cdm_status where status = 'condition' +--SELECT count(CONDITIONID) from condition where rownum = 1 \ No newline at end of file diff --git a/Oracle/death.sql b/Oracle/death.sql index da948d4..dc95680 100644 --- a/Oracle/death.sql +++ b/Oracle/death.sql @@ -1,7 +1,8 @@ --------------------------------------------------------------------------------- --- DEATH --------------------------------------------------------------------------------- +/** death - create and populate the death table. +*/ +select patid from demographic where 'dep' = 'demographic.sql' +/ BEGIN PMN_DROPSQL('DROP TABLE death'); END; @@ -45,9 +46,8 @@ where (pat.death_date is not null or vital_status_cd like 'Z%') and pat.patient_ end; / -BEGIN -PCORNetDeath(); -END; +insert into cdm_status (status, last_update) values ('death', sysdate) / ---SELECT count(PATID) from death where rownum = 1 -SELECT 1 FROM dual \ No newline at end of file +select 1 from cdm_status where status = 'death' + +--SELECT count(PATID) from death where rownum = 1 \ No newline at end of file diff --git a/Oracle/death_cause.sql b/Oracle/death_cause.sql index 40c14b7..ff8c3dd 100644 --- a/Oracle/death_cause.sql +++ b/Oracle/death_cause.sql @@ -1,7 +1,7 @@ --------------------------------------------------------------------------------- --- DEATH_CAUSE --------------------------------------------------------------------------------- - +/** death_cause - create the death_cause table. +*/ +select synonym_name from all_synonyms where 'dep' = 'pcornet_init.sql' +/ BEGIN PMN_DROPSQL('DROP TABLE death_cause'); END; @@ -15,5 +15,7 @@ CREATE TABLE death_cause( DEATH_CAUSE_CONFIDENCE varchar(2) NULL ) / ---SELECT count(PATID) from death_cause where rownum = 1 -SELECT count(MEDADMINID) from med_admin where rownum = 1 \ No newline at end of file +insert into cdm_status (status, last_update) values ('death_cause', sysdate) +/ +select 1 from cdm_status where status = 'death_cause' +--SELECT count(PATID) from death_cause where rownum = 1 \ No newline at end of file diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql index 72100c0..bb87d0d 100644 --- a/Oracle/demographic.sql +++ b/Oracle/demographic.sql @@ -1,7 +1,7 @@ --------------------------------------------------------------------------------- --- DEMOGRAPHIC --------------------------------------------------------------------------------- - +/** demographic - create and populate the demographic table. +*/ +select synonym_name from all_synonyms where 'dep' = 'pcornet_init.sql' +/ BEGIN PMN_DROPSQL('DROP TABLE demographic'); END; @@ -28,93 +28,149 @@ create or replace procedure PCORNetDemographic as sqltext varchar2(4000); cursor getsql is --1 -- S,R,NH - select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) - select '1', patient_num, birth_date, to_char(birth_date,''HH24:MI''), sex.pcori_basecode, 'NI', 'NI', 'NI', race.pcori_basecode - from i2b2patient p - where lower(p.sex_cd) in (lower(sex.c_dimcode)) - and lower(p.race_cd) in (lower(race.c_dimcode)) - and lower(nvl(p.ethnicity_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') - from pcornet_demo race, pcornet_demo sex - where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' - and race.c_visualattributes like 'L%' - and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' - and sex.c_visualattributes like 'L%' + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''1'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + ''''||sex.pcori_basecode||''','|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + ''''||race.pcori_basecode||''''|| + ' from i2b2patient p '|| + ' where lower(p.sex_cd) in ('||lower(sex.c_dimcode)||') '|| + ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| + ' and lower(nvl(p.ethnicity_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') ' + from pcornet_demo race, pcornet_demo sex + where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' + and race.c_visualattributes like 'L%' + and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' + and sex.c_visualattributes like 'L%' union -- A - S,R,H - select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) - select 'A', patient_num, birth_date, to_char(birth_date,''HH24:MI''), sex.pcori_basecode, 'NI', 'NI', hisp.pcori_basecode, race.pcori_basecode - from i2b2patient p - where lower(p.sex_cd) in (lower(sex.c_dimcode)) - and lower(p.race_cd) in (lower(race.c_dimcode)) - and lower(p.ethnicity_cd) in (lower(hisp.c_dimcode)) - from pcornet_demo race, pcornet_demo hisp, pcornet_demo sex - where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' - and race.c_visualattributes like 'L%' - and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' - and hisp.c_visualattributes like 'L%' - and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' - and sex.c_visualattributes like 'L%' +select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''A'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + ''''||sex.pcori_basecode||''','|| + '''NI'','|| + '''NI'','|| + ''''||hisp.pcori_basecode||''','|| + ''''||race.pcori_basecode||''''|| + ' from i2b2patient p '|| + ' where lower(p.sex_cd) in ('||lower(sex.c_dimcode)||') '|| + ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| + ' and lower(p.ethnicity_cd) in ('||lower(hisp.c_dimcode)||') ' + from pcornet_demo race, pcornet_demo hisp, pcornet_demo sex + where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' + and race.c_visualattributes like 'L%' + and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' + and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' + and sex.c_visualattributes like 'L%' union --2 S, nR, nH - select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) - select '2', patient_num, birth_date, to_char(birth_date,''HH24:MI''), sex.pcori_basecode, 'NI', 'NI', 'NI', 'NI' - from i2b2patient p - where lower(nvl(p.sex_cd,''xx'')) in (lower(sex.c_dimcode)) - and lower(nvl(p.race_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='RACE') - and lower(nvl(p.ethnicity_cd,'ni')) not in (select lower(code) from pcornet_codelist where codetype='HISPANIC') - from pcornet_demo sex - where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' - and sex.c_visualattributes like 'L%' + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''2'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + ''''||sex.pcori_basecode||''','|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + '''NI'''|| + ' from i2b2patient p '|| + ' where lower(nvl(p.sex_cd,''xx'')) in ('||lower(sex.c_dimcode)||') '|| + ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') '|| + ' and lower(nvl(p.ethnicity_cd,''ni'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') ' + from pcornet_demo sex + where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' + and sex.c_visualattributes like 'L%' union --3 -- nS,R, NH - select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) - select '3', patient_num, birth_date, to_char(birth_date,''HH24:MI''), 'NI', 'NI', 'NI', 'NI', race.pcori_basecode - from i2b2patient p - where lower(nvl(p.sex_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='SEX') - and lower(p.race_cd) in (lower(race.c_dimcode)) - and lower(nvl(p.ethnicity_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='HISPANIC') - from pcornet_demo race - where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' - and race.c_visualattributes like 'L%' + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''3'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + ''''||race.pcori_basecode||''''|| + ' from i2b2patient p '|| + ' where lower(nvl(p.sex_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''SEX'') '|| + ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| + ' and lower(nvl(p.ethnicity_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'')' + from pcornet_demo race + where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' + and race.c_visualattributes like 'L%' union --B -- nS,R, H - select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) - select 'B', patient_num, birth_date, to_char(birth_date,'HH24:MI'), 'NI', 'NI', 'NI', hisp.pcori_basecode, race.pcori_basecode - from i2b2patient p - where lower(nvl(p.sex_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='SEX') - and lower(p.race_cd) in (lower(race.c_dimcode)) - and lower(p.ethnicity_cd) in (lower(hisp.c_dimcode)) - from pcornet_demo race,pcornet_demo hisp - where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' - and race.c_visualattributes like 'L%' - and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' - and hisp.c_visualattributes like 'L%' + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''B'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + ''''||hisp.pcori_basecode||''','|| + ''''||race.pcori_basecode||''''|| + ' from i2b2patient p '|| + ' where lower(nvl(p.sex_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''SEX'') '|| + ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| + ' and lower(p.ethnicity_cd) in ('||lower(hisp.c_dimcode)||') ' + from pcornet_demo race,pcornet_demo hisp + where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' + and race.c_visualattributes like 'L%' + and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' union --4 -- S, NR, H - select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) - select '4', patient_num, birth_date, to_char(birth_date,''HH24:MI''), sex.pcori_basecode, 'NI', 'NI', hisp.pcori_basecode, 'NI' - from i2b2patient p - where lower(nvl(p.sex_cd,'NI')) in (lower(sex.c_dimcode)) - and lower(nvl(p.ethnicity_cd,'NI')) in (lower(hisp.c_dimcode)) - and lower(nvl(p.race_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='RACE') - from pcornet_demo sex, pcornet_demo hisp - where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' - and sex.c_visualattributes like 'L%' - and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' - and hisp.c_visualattributes like 'L%' + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''4'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + ''''||sex.pcori_basecode||''','|| + '''NI'','|| + '''NI'','|| + ''''||hisp.pcori_basecode||''','|| + '''NI'''|| + ' from i2b2patient p '|| + ' where lower(nvl(p.sex_cd,''NI'')) in ('||lower(sex.c_dimcode)||') '|| + ' and lower(nvl(p.ethnicity_cd,''NI'')) in ('||lower(hisp.c_dimcode)||') '|| + ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') ' + from pcornet_demo sex, pcornet_demo hisp + where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' + and sex.c_visualattributes like 'L%' + and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' union --5 -- NS, NR, H - select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) - select '5', patient_num, birth_date, to_char(birth_date,'HH24:MI'), 'NI', 'NI', 'NI', hisp.pcori_basecode, 'NI' - from i2b2patient p - where lower(nvl(p.sex_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='SEX') - and lower(nvl(p.race_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='RACE') - and lower(nvl(p.ethnicity_cd,'NI')) in (lower(hisp.c_dimcode)) - from pcornet_demo hisp - where hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' - and hisp.c_visualattributes like 'L%' + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''5'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + ''''||hisp.pcori_basecode||''','|| + '''NI'''|| + ' from i2b2patient p '|| + ' where lower(nvl(p.sex_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''SEX'') '|| + ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') '|| + ' and lower(nvl(p.ethnicity_cd,''NI'')) in ('||lower(hisp.c_dimcode)||') ' + from pcornet_demo hisp + where hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' union --6 -- NS, NR, nH - select insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) - select '6', patient_num, birth_date, to_char(birth_date,'HH24:MI'), 'NI', 'NI', 'NI', 'NI', 'NI' - from i2b2patient p - where lower(nvl(p.sex_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='SEX') - and lower(nvl(p.ethnicity_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='HISPANIC') - and lower(nvl(p.race_cd,'xx')) not in (select lower(code) from pcornet_codelist where codetype='RACE') - from dual; + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''6'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + '''NI'''|| + ' from i2b2patient p '|| + ' where lower(nvl(p.sex_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''SEX'') '|| + ' and lower(nvl(p.ethnicity_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') '|| + ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') ' + from dual; begin pcornet_popcodelist; @@ -134,13 +190,13 @@ END LOOP; CLOSE getsql; execute immediate 'create unique index demographic_pk on demographic (PATID)'; ---GATHER_TABLE_STATS('DEMOGRAPHIC'); +GATHER_TABLE_STATS('DEMOGRAPHIC'); end PCORNetDemographic; / -BEGIN -PCORNetDemographic(); -END; +insert into cdm_status (status, last_update) values ('demographic', sysdate) +--BEGIN +--PCORNetDemographic(); +--END; / ---SELECT count(PATID) from demographic where rownum = 1 -SELECT count(MEDADMINID) from med_admin where rownum = 1 \ No newline at end of file +select 1 from cdm_status where status = 'demographic' \ No newline at end of file diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 8b15c9f..2457fcf 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -1,7 +1,8 @@ --------------------------------------------------------------------------------- --- DIAGNOSIS --------------------------------------------------------------------------------- +/** diagnosis - create and populate the diagnosis table. +*/ +select encounterid from encounter where 'dep' = 'encounter.sql' +/ BEGIN PMN_DROPSQL('DROP TABLE diagnosis'); END; @@ -243,9 +244,10 @@ execute immediate 'create index diagnosis_idx on diagnosis (PATID, ENCOUNTERID)' end PCORNetDiagnosis; / -BEGIN -PCORNetDiagnosis(); -END; +insert into cdm_status (status, last_update) values ('diagnosis', sysdate) +--BEGIN +--PCORNetDiagnosis(); +--END; / -SELECT count(DIAGNOSISID) from diagnosis where rownum = 1 ---SELECT 1 FROM dual \ No newline at end of file +select 1 from cdm_status where status = 'diagnosis' +--SELECT count(DIAGNOSISID) from diagnosis where rownum = 1 \ No newline at end of file diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index 99e8de9..01e32a6 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -1,6 +1,7 @@ --------------------------------------------------------------------------------- --- DISPENSING --------------------------------------------------------------------------------- +/** dispensing - create and populate the dispensing table. +*/ + +--select encounterid from encounter where 'dep' = 'encounter.sql'; BEGIN PMN_DROPSQL('DROP TABLE dispensing'); @@ -176,9 +177,10 @@ execute immediate 'create index dispensing_idx on dispensing (PATID)'; end PCORNetDispensing; / -BEGIN -PCORNetDispensing(); -END; +insert into cdm_status (status, last_update) values ('dispensing', sysdate) +--BEGIN +--PCORNetDispensing(); +--END; / -SELECT count(DISPENSINGID) from dispensing where rownum = 1 ---SELECT 1 FROM dual \ No newline at end of file +select 1 from cdm_status where status = 'dispensing' +--SELECT count(DISPENSINGID) from dispensing where rownum = 1 \ No newline at end of file diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index fdfa4e4..824c0de 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -1,7 +1,8 @@ --------------------------------------------------------------------------------- --- ENCOUNTER --------------------------------------------------------------------------------- +/** encounter - create and populate the encounter table. +*/ +select patid from demographic where 'dep' = 'demographic.sql' +/ BEGIN PMN_DROPSQL('DROP TABLE encounter'); END; @@ -63,7 +64,7 @@ and enc.c_fullname like '\PCORI\ENCOUNTER\DRG\%') drg1 group by patient_num,enco where rn=1; execute immediate 'create index drg_idx on drg (patient_num, encounter_num)'; -GATHER_TABLE_STATS('drg'); +--GATHER_TABLE_STATS('drg'); insert into encounter(PATID,ENCOUNTERID,admit_date ,ADMIT_TIME , DISCHARGE_DATE ,DISCHARGE_TIME ,PROVIDERID ,FACILITY_LOCATION @@ -93,13 +94,11 @@ left outer join execute immediate 'create unique index encounter_pk on encounter (ENCOUNTERID)'; execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; -GATHER_TABLE_STATS('ENCOUNTER'); +--GATHER_TABLE_STATS('ENCOUNTER'); end PCORNetEncounter; / -BEGIN -PCORNetEncounter(); -END; +insert into cdm_status (status, last_update) values ('encounter', sysdate) / ---SELECT count(ENCOUNTERID) from encounter where rownum = 1 -SELECT 1 FROM dual \ No newline at end of file +select 1 from cdm_status where status = 'encounter' +--SELECT count(ENCOUNTERID) from encounter where rownum = 1 \ No newline at end of file diff --git a/Oracle/enrollment.sql b/Oracle/enrollment.sql index 4d75693..08f46d9 100644 --- a/Oracle/enrollment.sql +++ b/Oracle/enrollment.sql @@ -1,7 +1,7 @@ --------------------------------------------------------------------------------- --- ENROLLMENT --------------------------------------------------------------------------------- - +/** enrollment - create and populate the enrollment table. +*/ +select synonym_name from all_synonyms where 'dep' = 'pcornet_init.sql' +/ BEGIN PMN_DROPSQL('DROP TABLE enrollment'); END; @@ -44,13 +44,14 @@ join i2b2visit visit on enr.patient_num = visit.patient_num group by visit.patient_num; execute immediate 'create index enrollment_idx on enrollment (PATID)'; -GATHER_TABLE_STATS('ENROLLMENT'); +--GATHER_TABLE_STATS('ENROLLMENT'); end PCORNetEnroll; / -BEGIN -PCORNetEnroll(); -END; +insert into cdm_status (status, last_update) values ('enrollment', sysdate) +--BEGIN +--PCORNetEnroll(); +--END; / ---SELECT count(PATID) from enrollment where rownum = 1 -SELECT 1 FROM dual \ No newline at end of file +select 1 from cdm_status where status = 'enrollment' +--SELECT count(PATID) from enrollment where rownum = 1 \ No newline at end of file diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index dad37b8..c9883b3 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -1,7 +1,22 @@ --------------------------------------------------------------------------------- --- HARVEST --------------------------------------------------------------------------------- +/** prescribing - create and populate the prescribing table. +*/ +select conditionid from condition where 'dep' = 'condition.sql' +/ +select patid from death where 'dep' = 'death.sql' +/ +select diagnosisid from diagnosis where 'dep' = 'diagnosis.sql' +/ +select dispensingid from dispensing where 'dep' = 'dispensing.sql' +/ +select lab_result_cm_id from lab_result_cm where 'dep' = 'lab_result_cm.sql' +/ +select prescribingid from prescribing where 'dep' = 'prescribing.sql' +/ +select proceduresid from procedures where 'dep' = 'procedures.sql' +/ +select vitalid from vital where 'dep' = 'vital.sql' +/ BEGIN PMN_DROPSQL('DROP TABLE harvest'); END; @@ -74,9 +89,10 @@ INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART end PCORNetHarvest; / -BEGIN -PCORNetHarvest(); -END; +insert into cdm_status (status, last_update) values ('harvest', sysdate) +--BEGIN +--PCORNetHarvest(); +--END; / -SELECT count(NETWORKID) from Harvest where rownum = 1 ---SELECT 1 FROM dual \ No newline at end of file +select 1 from cdm_status where status = 'harvest' +--SELECT count(NETWORKID) from Harvest where rownum = 1 \ No newline at end of file diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 80efa99..9aefb6c 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -1,7 +1,8 @@ --------------------------------------------------------------------------------- --- LAB_RESULT_CM --------------------------------------------------------------------------------- +/** lab_result_cm - create and populate the lab_result_cm table. +*/ +select encounterid from encounter where 'dep' = 'encounter.sql' +/ BEGIN PMN_DROPSQL('DROP TABLE lab_result_cm'); END; @@ -235,9 +236,10 @@ execute immediate 'create index lab_result_cm_idx on lab_result_cm (PATID, ENCOU END PCORNetLabResultCM; / -BEGIN -PCORNetLabResultCM(); -END; +insert into cdm_status (status, last_update) values ('lab_result_cm', sysdate) +--BEGIN +--PCORNetLabResultCM(); +--END; / -SELECT count(LAB_RESULT_CM_ID) from lab_result_cm where rownum = 1 ---SELECT 1 FROM dual \ No newline at end of file +select 1 from cdm_status where status = 'lab_result_cm' +--SELECT count(LAB_RESULT_CM_ID) from lab_result_cm where rownum = 1 \ No newline at end of file diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 889ca38..035c691 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -1,30 +1,7 @@ --------------------------------------------------------------------------------- --- HELPER FUNCTIONS AND PROCEDURES --------------------------------------------------------------------------------- ---These helper functions also exist in PCORNetInit.sql. They are reproduced ---here to make the med_admin script an independent luigi job. ---TODO: consolidate helpers in a single, table independent job. -create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS - BEGIN - DBMS_STATS.GATHER_TABLE_STATS ( - ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. - tabname => table_name, - estimate_percent => 50, -- Percentage picked somewhat arbitrarily - cascade => TRUE, - degree => 16 - ); -END GATHER_TABLE_STATS; +/** med_admin - create and populate the med_admin table. +*/ +select synonym_name from all_synonyms where 'dep' = 'pcornet_init.sql' / -create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS - BEGIN - EXECUTE IMMEDIATE sqlstring; - EXCEPTION - WHEN OTHERS THEN NULL; -END PMN_DROPSQL; -/ --------------------------------------------------------------------------------- --- MED_ADMIN --------------------------------------------------------------------------------- BEGIN PMN_DROPSQL('DROP TABLE med_admin'); END; @@ -129,12 +106,13 @@ on med_p.c_basecode = med_start.concept_cd ; execute immediate 'create index med_admin_idx on med_admin (PATID, ENCOUNTERID)'; ---GATHER_TABLE_STATS('MED_ADMIN'); +GATHER_TABLE_STATS('MED_ADMIN'); end PCORNetMedAdmin; / -BEGIN -PCORNetMedAdmin(); -END; +insert into cdm_status (status, last_update) values ('med_admin', sysdate) +--BEGIN +--PCORNetMedAdmin(); +--END; / -SELECT count(MEDADMINID) from med_admin where rownum = 1 \ No newline at end of file +select 1 from cdm_status where status = 'med_admin' \ No newline at end of file diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 1ab9c38..32927a1 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -1,7 +1,8 @@ --------------------------------------------------------------------------------- --- HELPER FUNCTIONS AND PROCEDURES --------------------------------------------------------------------------------- +/** pcornet_init - create helper functions and procedures +*/ +alter session set current_schema = pcornet_cdm +/ create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS BEGIN DBMS_STATS.GATHER_TABLE_STATS ( @@ -165,55 +166,6 @@ end loop; close getcodesql ; end pcornet_popcodelist; / - - --------------------------------------------------------------------------------- --- I2B2 SYNONYMS, VIEWS, AND INTERMEDIARY TABLES --------------------------------------------------------------------------------- - - -CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT -/ - -CREATE OR REPLACE SYNONYM I2B2MEDFACT FOR OBSERVATION_FACT_MEDS -/ - -BEGIN -PMN_DROPSQL('DROP TABLE i2b2patient_list'); -END; -/ - -CREATE table i2b2patient_list as -select * from -( -select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') -) where ROWNUM<100000000 -/ - -create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) -/ - -create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE Environment: + return dict(datamart_id='C4UK', datamart_name='University of Kansas', i2b2_data_schema='BLUEHERONDATA', + min_pat_list_date_dd_mon_rrrr='01-Jan-2010', min_visit_date_dd_mon_rrrr='01-Jan-2010', + i2b2_meta_schema='BLUEHERONMETADATA', enrollment_months_back='2', network_id='C4', + network_name='GPC') + + +class condition(CDMScriptTask): script = Script.condition - def requires(self) -> List[luigi.Task]: - return [Encounter()] -class Death(SqlScriptTask): +class death(CDMScriptTask): script = Script.death - def requires(self) -> List[luigi.Task]: - return [Demographic()] -class DeathCause(SqlScriptTask): +class death_cause(CDMScriptTask): script = Script.death_cause -class Demographic(SqlScriptTask): + +class demographic(CDMScriptTask): script = Script.demographic - def requires(self) -> List[luigi.Task]: - return [PcornetInit()] -class Diagnosis(SqlScriptTask): +class diagnosis(CDMScriptTask): script = Script.diagnosis - def requires(self) -> List[luigi.Task]: - return [Encounter()] - - @property - def variables(self) -> Environment: - return dict(i2b2_meta_schema='BLUEHERONMETADATA') -class Dispensing(SqlScriptTask): +class dispensing(CDMScriptTask): script = Script.dispensing - def requires(self) -> List[luigi.Task]: - return [Encounter()] - @property - def variables(self) -> Environment: - return dict(i2b2_meta_schema='BLUEHERONMETADATA') - - -class Encounter(SqlScriptTask): +class encounter(CDMScriptTask): script = Script.encounter - def requires(self) -> List[luigi.Task]: - return [Demographic()] -class Enrollment(SqlScriptTask): +class enrollment(CDMScriptTask): script = Script.enrollment - @property - def variables(self) -> Environment: - return dict(enrollment_months_back='2') -class Harvest(SqlScriptTask): +class harvest(CDMScriptTask): script = Script.harvest - def requires(self) -> List[luigi.Task]: - return [Condition(), Death(), Diagnosis(), Dispensing(), LabResultCM(), Prescribing(), Procedures(), Vital()] - @property - def variables(self) -> Environment: - return dict(network_id='C4', network_name='GPC') - -class LabResultCM(SqlScriptTask): +class lab_result_cm(CDMScriptTask): script = Script.lab_result_cm - def requires(self) -> List[luigi.Task]: - return [Encounter()] -class MedAdmin(SqlScriptTask): +class med_admin(CDMScriptTask): script = Script.med_admin - def requires(self) -> List[luigi.Task]: - return [MedAdminInit()] -class MedAdminInit(SqlScriptTask): +class med_admin_init(CDMScriptTask): script = Script.med_admin_init -class ObsClin(SqlScriptTask): + +class obs_clin(CDMScriptTask): script = Script.obs_clin - def requires(self) -> List[luigi.Task]: - return [] -class ObsGen(SqlScriptTask): +class obs_gen(CDMScriptTask): script = Script.obs_gen - def requires(self) -> List[luigi.Task]: - return [] -class PcornetInit(SqlScriptTask): +class pcornet_init(CDMScriptTask): script = Script.pcornet_init - @property - def variables(self) -> Environment: - return dict(datamart_id='C4UK', datamart_name='University of Kansas', i2b2_data_schema='BLUEHERONDATA', min_pat_list_date_dd_mon_rrrr='01-Jan-2010', min_visit_date_dd_mon_rrrr='01-Jan-2010', - i2b2_meta_schema='BLUEHERONMETADATA') -class PcornetLoader(SqlScriptTask): +class pcornet_loader(CDMScriptTask): script = Script.pcornet_loader - def requires(self) -> List[luigi.Task]: - #return [DeathCause(), Enrollment(), Harvest(), MedAdmin(), PcornetTrail(), ProCM()] - return [DeathCause(), Demographic(), MedAdmin()] -class PcornetTrail(SqlScriptTask): - script = Script.pcornet_trail +class pcornet_trial(CDMScriptTask): + script = Script.pcornet_trial - def requires(self) -> List[luigi.Task]: - return [] -class Prescribing(SqlScriptTask): +class prescribing(CDMScriptTask): script = Script.prescribing - def requires(self) -> List[luigi.Task]: - return [Encounter()] -class ProCM(SqlScriptTask): +class pro_cm(CDMScriptTask): script = Script.pro_cm - def requires(self) -> List[luigi.Task]: - return [] -class Procedures(SqlScriptTask): +class procedures(CDMScriptTask): script = Script.procedures - def requires(self) -> List[luigi.Task]: - return [Encounter()] -class Provider(SqlScriptTask): +class provider(CDMScriptTask): script = Script.provider - def requires(self) -> List[luigi.Task]: - return [] -class Vital(SqlScriptTask): +class vital(CDMScriptTask): script = Script.vital - - def requires(self) -> List[luigi.Task]: - return [Encounter()] \ No newline at end of file diff --git a/script_lib.py b/script_lib.py index 70f6482..933292e 100644 --- a/script_lib.py +++ b/script_lib.py @@ -281,7 +281,7 @@ class Script(ScriptMixin, enum.Enum): obs_gen, pcornet_init, pcornet_loader, - pcornet_trail, + pcornet_trial, prescribing, pro_cm, procedures, @@ -307,7 +307,7 @@ class Script(ScriptMixin, enum.Enum): 'obs_gen.sql', 'pcornet_init.sql', 'pcornet_loader.sql', - 'pcornet_trail.sql', + 'pcornet_trial.sql', 'prescribing.sql', 'pro_cm.sql', 'procedures.sql', From 990fa15fd101f2f240f19c33d753b64cbcc88741 Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 5 Mar 2018 11:54:29 -0600 Subject: [PATCH 334/507] Luigi CDM full build with table create and populate --- Oracle/condition.sql | 9 +++++---- Oracle/death.sql | 4 ++++ Oracle/demographic.sql | 7 ++++--- Oracle/diagnosis.sql | 9 +++++---- Oracle/dispensing.sql | 9 +++++---- Oracle/encounter.sql | 6 +++++- Oracle/enrollment.sql | 9 +++++---- Oracle/harvest.sql | 9 +++++---- Oracle/lab_result_cm.sql | 9 +++++---- Oracle/med_admin.sql | 7 ++++--- Oracle/pcornet_init.sql | 3 --- Oracle/prescribing.sql | 9 +++++---- Oracle/procedures.sql | 9 +++++---- Oracle/vital.sql | 9 +++++---- 14 files changed, 62 insertions(+), 46 deletions(-) diff --git a/Oracle/condition.sql b/Oracle/condition.sql index cedd16f..0e417e8 100644 --- a/Oracle/condition.sql +++ b/Oracle/condition.sql @@ -97,14 +97,15 @@ log errors into ERR$_CONDITION reject limit unlimited ; execute immediate 'create index condition_idx on condition (PATID, ENCOUNTERID)'; ---GATHER_TABLE_STATS('CONDITION'); +GATHER_TABLE_STATS('CONDITION'); end PCORNetCondition; / +BEGIN +PCORNetCondition(); +END; +/ insert into cdm_status (status, last_update) values ('condition', sysdate) ---BEGIN ---PCORNetCondition(); ---END; / select 1 from cdm_status where status = 'condition' --SELECT count(CONDITIONID) from condition where rownum = 1 \ No newline at end of file diff --git a/Oracle/death.sql b/Oracle/death.sql index dc95680..ea8e8a3 100644 --- a/Oracle/death.sql +++ b/Oracle/death.sql @@ -46,6 +46,10 @@ where (pat.death_date is not null or vital_status_cd like 'Z%') and pat.patient_ end; / +BEGIN +PCORNetDeath(); +END; +/ insert into cdm_status (status, last_update) values ('death', sysdate) / select 1 from cdm_status where status = 'death' diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql index bb87d0d..53d24fe 100644 --- a/Oracle/demographic.sql +++ b/Oracle/demographic.sql @@ -194,9 +194,10 @@ GATHER_TABLE_STATS('DEMOGRAPHIC'); end PCORNetDemographic; / +BEGIN +PCORNetDemographic(); +END; +/ insert into cdm_status (status, last_update) values ('demographic', sysdate) ---BEGIN ---PCORNetDemographic(); ---END; / select 1 from cdm_status where status = 'demographic' \ No newline at end of file diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 2457fcf..4015aca 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -240,14 +240,15 @@ where (sourcefact.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\%' or so ; execute immediate 'create index diagnosis_idx on diagnosis (PATID, ENCOUNTERID)'; ---GATHER_TABLE_STATS('DIAGNOSIS'); +GATHER_TABLE_STATS('DIAGNOSIS'); end PCORNetDiagnosis; / +BEGIN +PCORNetDiagnosis(); +END; +/ insert into cdm_status (status, last_update) values ('diagnosis', sysdate) ---BEGIN ---PCORNetDiagnosis(); ---END; / select 1 from cdm_status where status = 'diagnosis' --SELECT count(DIAGNOSISID) from diagnosis where rownum = 1 \ No newline at end of file diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index 01e32a6..a0f4378 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -173,14 +173,15 @@ group by m.encounter_num ,m.patient_num, m.start_date, mo.pcori_ndc; */ execute immediate 'create index dispensing_idx on dispensing (PATID)'; ---GATHER_TABLE_STATS('DISPENSING'); +GATHER_TABLE_STATS('DISPENSING'); end PCORNetDispensing; / +BEGIN +PCORNetDispensing(); +END; +/ insert into cdm_status (status, last_update) values ('dispensing', sysdate) ---BEGIN ---PCORNetDispensing(); ---END; / select 1 from cdm_status where status = 'dispensing' --SELECT count(DISPENSINGID) from dispensing where rownum = 1 \ No newline at end of file diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 824c0de..274e31b 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -94,10 +94,14 @@ left outer join execute immediate 'create unique index encounter_pk on encounter (ENCOUNTERID)'; execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; ---GATHER_TABLE_STATS('ENCOUNTER'); +GATHER_TABLE_STATS('ENCOUNTER'); end PCORNetEncounter; / +BEGIN +PCORNetEncounter(); +END; +/ insert into cdm_status (status, last_update) values ('encounter', sysdate) / select 1 from cdm_status where status = 'encounter' diff --git a/Oracle/enrollment.sql b/Oracle/enrollment.sql index 08f46d9..23bee6a 100644 --- a/Oracle/enrollment.sql +++ b/Oracle/enrollment.sql @@ -44,14 +44,15 @@ join i2b2visit visit on enr.patient_num = visit.patient_num group by visit.patient_num; execute immediate 'create index enrollment_idx on enrollment (PATID)'; ---GATHER_TABLE_STATS('ENROLLMENT'); +GATHER_TABLE_STATS('ENROLLMENT'); end PCORNetEnroll; / +BEGIN +PCORNetEnroll(); +END; +/ insert into cdm_status (status, last_update) values ('enrollment', sysdate) ---BEGIN ---PCORNetEnroll(); ---END; / select 1 from cdm_status where status = 'enrollment' --SELECT count(PATID) from enrollment where rownum = 1 \ No newline at end of file diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index c9883b3..750c3cc 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -1,4 +1,4 @@ -/** prescribing - create and populate the prescribing table. +/** harvest - create and populate the harvest table. */ select conditionid from condition where 'dep' = 'condition.sql' @@ -89,10 +89,11 @@ INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART end PCORNetHarvest; / +BEGIN +PCORNetHarvest(); +END; +/ insert into cdm_status (status, last_update) values ('harvest', sysdate) ---BEGIN ---PCORNetHarvest(); ---END; / select 1 from cdm_status where status = 'harvest' --SELECT count(NETWORKID) from Harvest where rownum = 1 \ No newline at end of file diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 9aefb6c..20f69e6 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -232,14 +232,15 @@ and (m.nval_num is null or m.nval_num<=9999999) -- exclude lengths that exceed t ; execute immediate 'create index lab_result_cm_idx on lab_result_cm (PATID, ENCOUNTERID)'; ---GATHER_TABLE_STATS('LAB_RESULT_CM'); +GATHER_TABLE_STATS('LAB_RESULT_CM'); END PCORNetLabResultCM; / +BEGIN +PCORNetLabResultCM(); +END; +/ insert into cdm_status (status, last_update) values ('lab_result_cm', sysdate) ---BEGIN ---PCORNetLabResultCM(); ---END; / select 1 from cdm_status where status = 'lab_result_cm' --SELECT count(LAB_RESULT_CM_ID) from lab_result_cm where rownum = 1 \ No newline at end of file diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 035c691..3579c0e 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -110,9 +110,10 @@ GATHER_TABLE_STATS('MED_ADMIN'); end PCORNetMedAdmin; / +BEGIN +PCORNetMedAdmin(); +END; +/ insert into cdm_status (status, last_update) values ('med_admin', sysdate) ---BEGIN ---PCORNetMedAdmin(); ---END; / select 1 from cdm_status where status = 'med_admin' \ No newline at end of file diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 32927a1..55e6856 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -1,8 +1,5 @@ /** pcornet_init - create helper functions and procedures */ - -alter session set current_schema = pcornet_cdm -/ create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS BEGIN DBMS_STATS.GATHER_TABLE_STATS ( diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index d76dbfd..9b75ecd 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -259,14 +259,15 @@ inner join encounter enc on enc.encounterid = m.encounter_Num where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); execute immediate 'create index prescribing_idx on prescribing (PATID, ENCOUNTERID)'; ---GATHER_TABLE_STATS('PRESCRIBING'); +GATHER_TABLE_STATS('PRESCRIBING'); end PCORNetPrescribing; / +BEGIN +PCORNetPrescribing(); +END; +/ insert into cdm_status (status, last_update) values ('prescribing', sysdate) ---BEGIN ---PCORNetPrescribing(); ---END; / select 1 from cdm_status where status = 'prescribing' --SELECT count(PRESCRIBINGID) from prescribing where rownum = 1 \ No newline at end of file diff --git a/Oracle/procedures.sql b/Oracle/procedures.sql index e00ab60..7e36fb1 100644 --- a/Oracle/procedures.sql +++ b/Oracle/procedures.sql @@ -54,14 +54,15 @@ from i2b2fact fact where pr.c_fullname like '\PCORI\PROCEDURE\%'; execute immediate 'create index procedures_idx on procedures (PATID, ENCOUNTERID)'; ---GATHER_TABLE_STATS('PROCEDURES'); +GATHER_TABLE_STATS('PROCEDURES'); end PCORNetProcedure; / +BEGIN +PCORNetProcedure(); +END; +/ insert into cdm_status (status, last_update) values ('procedures', sysdate) ---BEGIN ---PCORNetProcedure(); ---END; / select 1 from cdm_status where status = 'procedures' --SELECT count(PATID) from procedures where rownum = 1 \ No newline at end of file diff --git a/Oracle/vital.sql b/Oracle/vital.sql index e32aa02..c6b86ea 100644 --- a/Oracle/vital.sql +++ b/Oracle/vital.sql @@ -130,14 +130,15 @@ where ht is not null group by patid, encounterid, measure_date, measure_time, admit_date) y; execute immediate 'create index vital_idx on vital (PATID)'; ---GATHER_TABLE_STATS('VITAL'); +GATHER_TABLE_STATS('VITAL'); end PCORNetVital; / +BEGIN +PCORNetVital(); +END; +/ insert into cdm_status (status, last_update) values ('vital', sysdate) ---BEGIN ---PCORNetVital(); ---END; / select 1 from cdm_status where status = 'vital' --SELECT count(VITALID) from vital where rownum = 1 From 4fc98801a42da29f8c69af7f0fde80d0205fb03c Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 6 Mar 2018 15:29:51 -0600 Subject: [PATCH 335/507] Added missing operations to pcornet_init --- Oracle/pcornet_init.sql | 47 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 55e6856..395fb1e 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -163,6 +163,53 @@ end loop; close getcodesql ; end pcornet_popcodelist; / + +CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT +/ + +CREATE OR REPLACE SYNONYM I2B2MEDFACT FOR OBSERVATION_FACT_MEDS +/ + +BEGIN +PMN_DROPSQL('DROP TABLE i2b2patient_list'); +END; +/ + +CREATE table i2b2patient_list as +select * from +( +select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') +) where ROWNUM<100000000 +/ + +create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) +/ + +create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE Date: Fri, 9 Mar 2018 11:41:14 -0600 Subject: [PATCH 336/507] Updated Harvest to CDM 4, added counts to status table. --- Oracle/condition.sql | 5 ++-- Oracle/death.sql | 2 +- Oracle/death_cause.sql | 2 +- Oracle/demographic.sql | 2 +- Oracle/diagnosis.sql | 2 +- Oracle/dispensing.sql | 2 +- Oracle/encounter.sql | 2 +- Oracle/enrollment.sql | 2 +- Oracle/harvest.sql | 52 ++++++++++++++++++++++++++++++++++----- Oracle/harvest_local.csv | 4 +-- Oracle/lab_result_cm.sql | 2 +- Oracle/med_admin.sql | 2 +- Oracle/obs_clin.sql | 12 +++++---- Oracle/obs_gen.sql | 10 +++++--- Oracle/pcornet_loader.sql | 10 -------- Oracle/pcornet_trial.sql | 2 +- Oracle/prescribing.sql | 12 ++++----- Oracle/pro_cm.sql | 2 +- Oracle/procedures.sql | 2 +- Oracle/provider.sql | 5 +++- Oracle/vital.sql | 2 +- i2p_tasks.py | 36 +++++++++++++++++++++++++++ script_lib.py | 2 ++ 23 files changed, 124 insertions(+), 50 deletions(-) diff --git a/Oracle/condition.sql b/Oracle/condition.sql index 0e417e8..de57c8f 100644 --- a/Oracle/condition.sql +++ b/Oracle/condition.sql @@ -105,7 +105,6 @@ BEGIN PCORNetCondition(); END; / -insert into cdm_status (status, last_update) values ('condition', sysdate) +insert into cdm_status (status, last_update, records) select 'condition', sysdate, count(*) from condition / -select 1 from cdm_status where status = 'condition' ---SELECT count(CONDITIONID) from condition where rownum = 1 \ No newline at end of file +select 1 from cdm_status where status = 'condition' \ No newline at end of file diff --git a/Oracle/death.sql b/Oracle/death.sql index ea8e8a3..4eb3e5b 100644 --- a/Oracle/death.sql +++ b/Oracle/death.sql @@ -50,7 +50,7 @@ BEGIN PCORNetDeath(); END; / -insert into cdm_status (status, last_update) values ('death', sysdate) +insert into cdm_status (status, last_update, records) select 'death', sysdate, count(*) from death / select 1 from cdm_status where status = 'death' diff --git a/Oracle/death_cause.sql b/Oracle/death_cause.sql index ff8c3dd..120608e 100644 --- a/Oracle/death_cause.sql +++ b/Oracle/death_cause.sql @@ -15,7 +15,7 @@ CREATE TABLE death_cause( DEATH_CAUSE_CONFIDENCE varchar(2) NULL ) / -insert into cdm_status (status, last_update) values ('death_cause', sysdate) +insert into cdm_status (status, last_update, records) select 'death_cause', sysdate, count(*) from death_cause / select 1 from cdm_status where status = 'death_cause' --SELECT count(PATID) from death_cause where rownum = 1 \ No newline at end of file diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql index 53d24fe..70d78f4 100644 --- a/Oracle/demographic.sql +++ b/Oracle/demographic.sql @@ -198,6 +198,6 @@ BEGIN PCORNetDemographic(); END; / -insert into cdm_status (status, last_update) values ('demographic', sysdate) +insert into cdm_status (status, last_update, records) select 'demographic', sysdate, count(*) from demographic / select 1 from cdm_status where status = 'demographic' \ No newline at end of file diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 4015aca..71979be 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -248,7 +248,7 @@ BEGIN PCORNetDiagnosis(); END; / -insert into cdm_status (status, last_update) values ('diagnosis', sysdate) +insert into cdm_status (status, last_update, records) select 'diagnosis', sysdate, count(*) from diagnosis / select 1 from cdm_status where status = 'diagnosis' --SELECT count(DIAGNOSISID) from diagnosis where rownum = 1 \ No newline at end of file diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index a0f4378..1331b63 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -181,7 +181,7 @@ BEGIN PCORNetDispensing(); END; / -insert into cdm_status (status, last_update) values ('dispensing', sysdate) +insert into cdm_status (status, last_update, records) select 'dispensing', sysdate, count(*) from dispensing / select 1 from cdm_status where status = 'dispensing' --SELECT count(DISPENSINGID) from dispensing where rownum = 1 \ No newline at end of file diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 274e31b..b0f24e1 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -102,7 +102,7 @@ BEGIN PCORNetEncounter(); END; / -insert into cdm_status (status, last_update) values ('encounter', sysdate) +insert into cdm_status (status, last_update, records) select 'encounter', sysdate, count(*) from encounter / select 1 from cdm_status where status = 'encounter' --SELECT count(ENCOUNTERID) from encounter where rownum = 1 \ No newline at end of file diff --git a/Oracle/enrollment.sql b/Oracle/enrollment.sql index 23bee6a..7c5084f 100644 --- a/Oracle/enrollment.sql +++ b/Oracle/enrollment.sql @@ -52,7 +52,7 @@ BEGIN PCORNetEnroll(); END; / -insert into cdm_status (status, last_update) values ('enrollment', sysdate) +insert into cdm_status (status, last_update, records) select 'enrollment', sysdate, count(*) from enrollment / select 1 from cdm_status where status = 'enrollment' --SELECT count(PATID) from enrollment where rownum = 1 \ No newline at end of file diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index 750c3cc..ca0d590 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -1,20 +1,35 @@ /** harvest - create and populate the harvest table. */ - select conditionid from condition where 'dep' = 'condition.sql' / select patid from death where 'dep' = 'death.sql' / +select patid from death_cause where 'dep' = 'death_cause.sql' +/ select diagnosisid from diagnosis where 'dep' = 'diagnosis.sql' / select dispensingid from dispensing where 'dep' = 'dispensing.sql' / +select patid from enrollment where 'dep' = 'enrollment.sql' +/ select lab_result_cm_id from lab_result_cm where 'dep' = 'lab_result_cm.sql' / +select medadminid from med_admin where 'dep' = 'med_admin.sql' +/ +select obsclinid from obs_clin where 'dep' = 'obs_clin.sql' +/ +select obsgenid from obs_gen where 'dep' = 'obs_gen.sql' +/ +select patid from pcornet_trial where 'dep' = 'pcornet_trial.sql' +/ select prescribingid from prescribing where 'dep' = 'prescribing.sql' / +select pro_cm_id from pro_cm where 'dep' = 'pro_cm.sql' +/ select proceduresid from procedures where 'dep' = 'procedures.sql' / +select providerid from provider where 'dep' = 'provider.sql' +/ select vitalid from vital where 'dep' = 'vital.sql' / BEGIN @@ -48,6 +63,11 @@ CREATE TABLE harvest( REPORT_DATE_MGMT varchar(2) NULL, RESOLVE_DATE_MGMT varchar(2) NULL, PRO_DATE_MGMT varchar(2) NULL, + DEATH_DATE_MGTM varchar(2) NULL, + MEDADMIN_START_DATE_MGMT varchar(2) NULL, + MEDADMIN_END_DATE_MGMT varchar(2) NULL, + OBSCLIN_DATE_MGMT varchar(2) NULL, + OBSGEN_DATE_MGMT varchar(2) NULL, REFRESH_DEMOGRAPHIC_DATE date NULL, REFRESH_ENROLLMENT_DATE date NULL, REFRESH_ENCOUNTER_DATE date NULL, @@ -61,7 +81,11 @@ CREATE TABLE harvest( REFRESH_PRESCRIBING_DATE date NULL, REFRESH_PCORNET_TRIAL_DATE date NULL, REFRESH_DEATH_DATE date NULL, - REFRESH_DEATH_CAUSE_DATE date NULL + REFRESH_DEATH_CAUSE_DATE date NULL, + REFRESH_MED_ADMIN_DATE date NULL, + REFRESH_OBS_CLIN_DATE date NULL, + REFRESH_PROVIDER_DATE date NULL, + REFRESH_OBS_GEN_DATE date NULL ) / create or replace procedure PCORNetHarvest as @@ -69,8 +93,19 @@ begin execute immediate 'truncate table harvest'; -INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) - select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.PX_DATE_MGMT, hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, +INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, + BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, + RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, + ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, DEATH_DATE_MGTM, MEDADMIN_START_DATE_MGMT, MEDADMIN_END_DATE_MGMT, + OBSCLIN_DATE_MGMT, OBSGEN_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, + REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, + REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, + REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE, REFRESH_MED_ADMIN_DATE, REFRESH_OBS_CLIN_DATE, REFRESH_PROVIDER_DATE, REFRESH_OBS_GEN_DATE) + select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, + hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.PX_DATE_MGMT, + hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, + hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, + hl.DEATH_DATE_MGMT, hl.MEDADMIN_START_DATE_MGMT, hl.MEDADMIN_END_DATE_MGMT, hl.OBSCLIN_DATE_MGMT, hl.OBSGEN_DATE_MGMT, case when (select count(*) from demographic) > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, case when (select count(*) from enrollment) > 0 then current_date else null end REFRESH_ENROLLMENT_DATE, case when (select count(*) from encounter) > 0 then current_date else null end REFRESH_ENCOUNTER_DATE, @@ -84,7 +119,12 @@ INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART case when (select count(*) from prescribing) > 0 then current_date else null end REFRESH_PRESCRIBING_DATE, case when (select count(*) from pcornet_trial) > 0 then current_date else null end REFRESH_PCORNET_TRIAL_DATE, case when (select count(*) from death) > 0 then current_date else null end REFRESH_DEATH_DATE, - case when (select count(*) from death_cause) > 0 then current_date else null end REFRESH_DEATH_CAUSE_DATE + case when (select count(*) from death_cause) > 0 then current_date else null end REFRESH_DEATH_CAUSE_DATE, + case when (select count(*) from med_admin) > 0 then current_date else null end REFRESH_MED_ADMIN_DATE, + case when (select count(*) from obs_clin) > 0 then current_date else null end REFRESH_OBS_CLIN_DATE, + case when (select count(*) from provider) > 0 then current_date else null end REFRESH_PROVIDER_DATE, + case when (select count(*) from obs_gen) > 0 then current_date else null end REFRESH_OBS_GEN_DATE + from harvest_local hl; end PCORNetHarvest; @@ -93,7 +133,7 @@ BEGIN PCORNetHarvest(); END; / -insert into cdm_status (status, last_update) values ('harvest', sysdate) +insert into cdm_status (status, last_update, records) select 'harvest', sysdate, count(*) from harvest / select 1 from cdm_status where status = 'harvest' --SELECT count(NETWORKID) from Harvest where rownum = 1 \ No newline at end of file diff --git a/Oracle/harvest_local.csv b/Oracle/harvest_local.csv index e20719b..e428cb6 100644 --- a/Oracle/harvest_local.csv +++ b/Oracle/harvest_local.csv @@ -1,2 +1,2 @@ -"DATAMART_CLAIMS","DATAMART_EHR","BIRTH_DATE_MGMT","ENR_START_DATE_MGMT","ENR_END_DATE_MGMT","ADMIT_DATE_MGMT","DISCHARGE_DATE_MGMT","PX_DATE_MGMT","RX_ORDER_DATE_MGMT","RX_START_DATE_MGMT","RX_END_DATE_MGMT","DISPENSE_DATE_MGMT","LAB_ORDER_DATE_MGMT","SPECIMEN_DATE_MGMT","RESULT_DATE_MGMT","MEASURE_DATE_MGMT","ONSET_DATE_MGMT","REPORT_DATE_MGMT","RESOLVE_DATE_MGMT","PRO_DATE_MGMT" -"01","02","03","03","03","03","03","03","03","03","04","03","03","03","03","03","03","03","03","03" +"DATAMART_CLAIMS","DATAMART_EHR","BIRTH_DATE_MGMT","ENR_START_DATE_MGMT","ENR_END_DATE_MGMT","ADMIT_DATE_MGMT","DISCHARGE_DATE_MGMT","PX_DATE_MGMT","RX_ORDER_DATE_MGMT","RX_START_DATE_MGMT","RX_END_DATE_MGMT","DISPENSE_DATE_MGMT","LAB_ORDER_DATE_MGMT","SPECIMEN_DATE_MGMT","RESULT_DATE_MGMT","MEASURE_DATE_MGMT","ONSET_DATE_MGMT","REPORT_DATE_MGMT","RESOLVE_DATE_MGMT","PRO_DATE_MGMT","DEATH_DATE_MGTM","MEDADMIN_START_DATE_MGMT","MEDADMIN_END_DATE_MGMT","OBSCLIN_DATE_MGMT","OBSGEN_DATE_MGMT" +"01","02","03","03","03","03","03","03","03","03","04","03","03","03","03","03","03","03","03","03","03","03","03","03","03" diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 20f69e6..555560e 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -240,7 +240,7 @@ BEGIN PCORNetLabResultCM(); END; / -insert into cdm_status (status, last_update) values ('lab_result_cm', sysdate) +insert into cdm_status (status, last_update, records) select 'lab_result_cm', sysdate, count(*) from lab_result_cm / select 1 from cdm_status where status = 'lab_result_cm' --SELECT count(LAB_RESULT_CM_ID) from lab_result_cm where rownum = 1 \ No newline at end of file diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 3579c0e..bd03536 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -114,6 +114,6 @@ BEGIN PCORNetMedAdmin(); END; / -insert into cdm_status (status, last_update) values ('med_admin', sysdate) +insert into cdm_status (status, last_update, records) select 'med_admin', sysdate, count(*) from med_admin / select 1 from cdm_status where status = 'med_admin' \ No newline at end of file diff --git a/Oracle/obs_clin.sql b/Oracle/obs_clin.sql index 813e118..f46e499 100644 --- a/Oracle/obs_clin.sql +++ b/Oracle/obs_clin.sql @@ -1,6 +1,5 @@ --------------------------------------------------------------------------------- --- OBS_CLIN --------------------------------------------------------------------------------- +/** obs_gen - create the obs_clin table. +*/ BEGIN PMN_DROPSQL('DROP TABLE obs_clin'); @@ -17,7 +16,7 @@ CREATE TABLE obs_clin( OBSCLIN_CODE varchar(50) NULL, OBSCLIN_RESULT_QUAL varchar(50) NULL, OBSCLIN_RESULT_TEXT varchar(50) NULL, - OBSCLIN_RESULT_SNOMED varchar(50), + OBSCLIN_RESULT_SNOMED varchar(50) NULL, OBSCLIN_RESULT_NUM NUMBER(18, 0) NULL, -- (8,0) OBSCLIN_RESULT_MODIFIER varchar(2) NULL, OBSCLIN_RESULT_UNIT varchar(50) NULL, @@ -28,4 +27,7 @@ CREATE TABLE obs_clin( RAW_OBSCLIN_MODIFIER varchar(50) NULL, RAW_OBSCLIN_UNIT varchar(50) NULL ) -/ \ No newline at end of file +/ +insert into cdm_status (status, last_update, records) select 'obs_clin', sysdate, count(*) from obs_clin +/ +select 1 from cdm_status where status = 'obs_clin' \ No newline at end of file diff --git a/Oracle/obs_gen.sql b/Oracle/obs_gen.sql index e1c83fc..0e2bef4 100644 --- a/Oracle/obs_gen.sql +++ b/Oracle/obs_gen.sql @@ -1,6 +1,5 @@ --------------------------------------------------------------------------------- --- OBS_GEN --------------------------------------------------------------------------------- +/** obs_gen - create the obs_gen table. +*/ BEGIN PMN_DROPSQL('DROP TABLE obs_gen'); @@ -28,4 +27,7 @@ CREATE TABLE obs_gen( RAW_OBSGEN_RESULT varchar(50) NULL, RAW_OBSGEN_UNIT varchar(50) NULL ) -/ \ No newline at end of file +/ +insert into cdm_status (status, last_update, records) select 'obs_gen', sysdate, count(*) from obs_gen +/ +select 1 from cdm_status where status = 'obs_gen' \ No newline at end of file diff --git a/Oracle/pcornet_loader.sql b/Oracle/pcornet_loader.sql index e89cc46..213e2fa 100644 --- a/Oracle/pcornet_loader.sql +++ b/Oracle/pcornet_loader.sql @@ -1,15 +1,5 @@ -select patid from death_cause where 'dep' = 'death_cause.sql' -/ -select patid from enrollment where 'dep' = 'enrollment.sql' -/ select networkid from harvest where 'dep' = 'harvest.sql' / -select medadminid from med_admin where 'dep' = 'med_admin.sql' -/ -select patid from pcornet_trial where 'dep' = 'pcornet_trial.sql' -/ -select pro_cm_id from pro_cm where 'dep' = 'pro_cm.sql' -/ insert into cdm_status (status, last_update) values ('pcornet_loader', sysdate ) / select 1 from cdm_status where status = 'pcornet_loader' \ No newline at end of file diff --git a/Oracle/pcornet_trial.sql b/Oracle/pcornet_trial.sql index 1ef42fc..6f35af5 100644 --- a/Oracle/pcornet_trial.sql +++ b/Oracle/pcornet_trial.sql @@ -17,6 +17,6 @@ CREATE TABLE pcornet_trial( TRIAL_INVITE_CODE varchar(20) NULL ) / -insert into cdm_status (status, last_update) values ('pcornet_trial', sysdate) +insert into cdm_status (status, last_update, records) select 'pcornet_trial', sysdate, count(*) from pcornet_trial / select 1 from cdm_status where status = 'pcornet_trial' \ No newline at end of file diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 9b75ecd..f9a6ca4 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -150,7 +150,7 @@ select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd, on basis.modifier_cd = basiscode.c_basecode and basiscode.c_fullname like '\PCORI_MOD\RX_BASIS\%'; -execute immediate 'create unique index basis_idx on basis (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; +execute immediate 'create index basis_idx on basis (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('BASIS'); insert into freq @@ -160,7 +160,7 @@ select pcori_basecode,instance_num,start_date,provider_id,concept_cd,encounter_n on freq.modifier_cd = freqcode.c_basecode and freqcode.c_fullname like '\PCORI_MOD\RX_FREQUENCY\%'; -execute immediate 'create unique index freq_idx on freq (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; +execute immediate 'create index freq_idx on freq (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('FREQ'); insert into quantity @@ -170,7 +170,7 @@ select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,mod on quantity.modifier_cd = quantitycode.c_basecode and quantitycode.c_fullname like '\PCORI_MOD\RX_QUANTITY\'; -execute immediate 'create unique index quantity_idx on quantity (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; +execute immediate 'create index quantity_idx on quantity (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('QUANTITY'); insert into refills @@ -180,7 +180,7 @@ select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,mod on refills.modifier_cd = refillscode.c_basecode and refillscode.c_fullname like '\PCORI_MOD\RX_REFILLS\'; -execute immediate 'create unique index refills_idx on refills (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; +execute immediate 'create index refills_idx on refills (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('REFILLS'); insert into supply @@ -190,7 +190,7 @@ select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,mod on supply.modifier_cd = supplycode.c_basecode and supplycode.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\'; -execute immediate 'create unique index supply_idx on supply (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; +execute immediate 'create index supply_idx on supply (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('SUPPLY'); -- insert data with outer joins to ensure all records are included even if some data elements are missing @@ -267,7 +267,7 @@ BEGIN PCORNetPrescribing(); END; / -insert into cdm_status (status, last_update) values ('prescribing', sysdate) +insert into cdm_status (status, last_update, records) select 'prescribing', sysdate, count(*) from prescribing / select 1 from cdm_status where status = 'prescribing' --SELECT count(PRESCRIBINGID) from prescribing where rownum = 1 \ No newline at end of file diff --git a/Oracle/pro_cm.sql b/Oracle/pro_cm.sql index 432c81a..03c1c57 100644 --- a/Oracle/pro_cm.sql +++ b/Oracle/pro_cm.sql @@ -37,6 +37,6 @@ begin select pro_cm_seq.nextval into :new.PRO_CM_ID from dual; end; / -insert into cdm_status (status, last_update) values ('pro_cm', sysdate) +insert into cdm_status (status, last_update, records) select 'pro_cm', sysdate, count(*) from pro_cm / select 1 from cdm_status where status = 'pro_cm' diff --git a/Oracle/procedures.sql b/Oracle/procedures.sql index 7e36fb1..3e47788 100644 --- a/Oracle/procedures.sql +++ b/Oracle/procedures.sql @@ -62,7 +62,7 @@ BEGIN PCORNetProcedure(); END; / -insert into cdm_status (status, last_update) values ('procedures', sysdate) +insert into cdm_status (status, last_update, records) select 'procedures', sysdate, count(*) from procedures / select 1 from cdm_status where status = 'procedures' --SELECT count(PATID) from procedures where rownum = 1 \ No newline at end of file diff --git a/Oracle/provider.sql b/Oracle/provider.sql index c8e5d55..22a42e7 100644 --- a/Oracle/provider.sql +++ b/Oracle/provider.sql @@ -40,7 +40,10 @@ insert into provider(provider_sex, provider_specialty_primary, provider_npi, pro ; execute immediate 'create index provider_idx on provider (PROVIDERID)'; ---GATHER_TABLE_STATS('PROVIDER'); +GATHER_TABLE_STATS('PROVIDER'); end PCORNetProvider; / +insert into cdm_status (status, last_update, records) select 'provider', sysdate, count(*) from provider +/ +select 1 from cdm_status where status = 'provider' diff --git a/Oracle/vital.sql b/Oracle/vital.sql index c6b86ea..5873cb9 100644 --- a/Oracle/vital.sql +++ b/Oracle/vital.sql @@ -138,7 +138,7 @@ BEGIN PCORNetVital(); END; / -insert into cdm_status (status, last_update) values ('vital', sysdate) +insert into cdm_status (status, last_update, records) select 'vital', sysdate, count(*) from vital / select 1 from cdm_status where status = 'vital' --SELECT count(VITALID) from vital where rownum = 1 diff --git a/i2p_tasks.py b/i2p_tasks.py index e8af2b5..28d053f 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -2,8 +2,12 @@ """ from etl_tasks import SqlScriptTask +from param_val import IntParam from script_lib import Script from sql_syntax import Environment +from sqlalchemy.engine import RowProxy +from sqlalchemy.exc import DatabaseError +from typing import List class CDMScriptTask(SqlScriptTask): @@ -71,6 +75,38 @@ class obs_gen(CDMScriptTask): script = Script.obs_gen +class patient_chunks_survey(SqlScriptTask): + script = Script.patient_chunks_survey + patient_chunks = IntParam(default=200) + patient_chunk_max = IntParam(default=None) + + #def variables(self) -> Environment: + # return dict(chunk_qty=str(self.patient_chunks)) + + #def run(self) -> None: + # SqlScriptTask.run_bound(self, script_params=dict(chunk_qty=str(self.patient_chunks)) + + def results(self) -> List[RowProxy]: + with self.connection(event='survey results') as lc: + q = ''' + select patient_num + , patient_num_qty + , patient_num_first + , patient_num_last + from patient_chunks + where chunk_qty = :chunk_qty + and (:chunk_max is null or + chunk_num <= :chunk_max) + order by chunk_num + ''' + params = dict(chunk_max=self.patient_chunk_max, chunk_qty=self.patient_chunks) + + try: + return lc.execute(q, params=params).fetchall() + except DatabaseError: + return [] + + class pcornet_init(CDMScriptTask): script = Script.pcornet_init diff --git a/script_lib.py b/script_lib.py index 933292e..989a4da 100644 --- a/script_lib.py +++ b/script_lib.py @@ -279,6 +279,7 @@ class Script(ScriptMixin, enum.Enum): med_admin_init, obs_clin, obs_gen, + patient_chunks_survey, pcornet_init, pcornet_loader, pcornet_trial, @@ -305,6 +306,7 @@ class Script(ScriptMixin, enum.Enum): 'med_admin_init.sql', 'obs_clin.sql', 'obs_gen.sql', + 'patient_chunks_survey.sql', 'pcornet_init.sql', 'pcornet_loader.sql', 'pcornet_trial.sql', From 0740ccb545a1f7d7daab069df64d8839d6a77b4d Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 9 Mar 2018 11:59:07 -0600 Subject: [PATCH 337/507] Add patient_chunks_survey.sql --- Oracle/patient_chunks_survey.sql | 49 ++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 Oracle/patient_chunks_survey.sql diff --git a/Oracle/patient_chunks_survey.sql b/Oracle/patient_chunks_survey.sql new file mode 100644 index 0000000..51058d0 --- /dev/null +++ b/Oracle/patient_chunks_survey.sql @@ -0,0 +1,49 @@ +/** patient_chunk_survery - create table to save ntile over patient_num +*/ + +whenever sqlerror continue; + drop table patient_chunks; +whenever sqlerror exit; +/ +create table patient_chunks ( + chunk_qty integer not null, + chunk_num integer not null, + patient_num_qty integer not null, + patient_num_first varchar2(64), + patient_num_last varchar2(64), + constraint patient_chunks_pk primary key (chunk_qty, chunk_num), + constraint chunk_qty_pos check (chunk_qty > 0), + constraint chunk_num_in_range check (chunk_num between 1 and chunk_qty), + constraint chunk_first check (patient_num_first is not null or chunk_num = 1) + ); + +-- Can we refer to the table without error? +--select coalesce((select 1 from patient_chunks where patient_id_qty > 0 and rownum=1), 1) complete +--from dual; +/ +insert into patient_chunks ( + chunk_qty + , chunk_num + , patient_num_qty + , patient_num_first + , patient_num_last +) +select :chunk_qty + , chunk_num + , count(distinct patient_num) patient_num_qty + , min(patient_num) patient_num_first + , max(patient_num) patient_num_last + from ( + select patient_num, ntile(:chunk_qty) over (order by patient_num) as chunk_num + from (select distinct patient_num from BLUEHERONDATA.patient_dimension)) +group by chunk_num, :chunk_qty +order by chunk_num; +/ +select case + when (select count(distinct chunk_num) + from patient_chunks + where chunk_qty = &&chunk_qty) = &&chunk_qty + then 1 + else 0 + end complete +from dual; From f33e2ad4ee2ad4fff53dac1d337cf1ca7e2ed5f4 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 28 Mar 2018 10:09:51 -0500 Subject: [PATCH 338/507] All PATID and ENCOUNTERID columns to number --- Oracle/prescribing.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index f9a6ca4..34d6b68 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -194,7 +194,7 @@ execute immediate 'create index supply_idx on supply (instance_num, start_date, GATHER_TABLE_STATS('SUPPLY'); -- insert data with outer joins to ensure all records are included even if some data elements are missing -insert into prescribing ( +insert /*+ parallel(10) */ into prescribing ( PATID ,encounterid ,RX_PROVIDERID From 7eef8f71fb6e44a5209bb793b12967b26ed2aef2 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 28 Mar 2018 12:10:46 -0500 Subject: [PATCH 339/507] Adding commits and parallel execution hints to reduce TEMP space --- Oracle/prescribing.sql | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 34d6b68..48b4056 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -150,6 +150,8 @@ select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd, on basis.modifier_cd = basiscode.c_basecode and basiscode.c_fullname like '\PCORI_MOD\RX_BASIS\%'; +commit; + execute immediate 'create index basis_idx on basis (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('BASIS'); @@ -160,6 +162,8 @@ select pcori_basecode,instance_num,start_date,provider_id,concept_cd,encounter_n on freq.modifier_cd = freqcode.c_basecode and freqcode.c_fullname like '\PCORI_MOD\RX_FREQUENCY\%'; +commit; + execute immediate 'create index freq_idx on freq (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('FREQ'); @@ -170,6 +174,8 @@ select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,mod on quantity.modifier_cd = quantitycode.c_basecode and quantitycode.c_fullname like '\PCORI_MOD\RX_QUANTITY\'; +commit; + execute immediate 'create index quantity_idx on quantity (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('QUANTITY'); @@ -180,6 +186,8 @@ select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,mod on refills.modifier_cd = refillscode.c_basecode and refillscode.c_fullname like '\PCORI_MOD\RX_REFILLS\'; +commit; + execute immediate 'create index refills_idx on refills (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('REFILLS'); @@ -190,6 +198,8 @@ select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,mod on supply.modifier_cd = supplycode.c_basecode and supplycode.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\'; +commit; + execute immediate 'create index supply_idx on supply (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('SUPPLY'); @@ -213,7 +223,7 @@ insert /*+ parallel(10) */ into prescribing ( -- ,RAW_RX_FREQUENCY, ,RAW_RXNORM_CUI ) -select distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH24:MI'), m.start_date start_date, m.end_date, mo.pcori_cui +select /*+ parallel(10) */ distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH24:MI'), m.start_date start_date, m.end_date, mo.pcori_cui ,quantity.nval_num quantity, 'NI' rx_quantity_unit, refills.nval_num refills, supply.nval_num supply, substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) frequency, substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) basis , substr(mo.c_name, 1, 50) raw_rx_med_name, substr(mo.c_basecode, 1, 50) raw_rxnorm_cui From 1d04f97af52157d3d73b5591d302600205d3691a Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 30 Mar 2018 12:54:45 -0500 Subject: [PATCH 340/507] Prescribing using temporary table --- Oracle/prescribing.sql | 83 ++++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 32 deletions(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 48b4056..bc62b16 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -17,7 +17,7 @@ CREATE TABLE prescribing( RX_START_DATE date NULL, RX_END_DATE date NULL, RX_QUANTITY number(18,5) NULL, - RX_QUANTITY_UNIT varchar(2) NULL, + RX_QUANTITY_UNIT varchar(2) NULL, RX_REFILLS number(18,5) NULL, RX_DAYS_SUPPLY number (18,5) NULL, RX_FREQUENCY varchar(2) NULL, @@ -25,8 +25,8 @@ CREATE TABLE prescribing( RXNORM_CUI varchar(8) NULL, RAW_RX_MED_NAME varchar (50) NULL, RAW_RX_FREQUENCY varchar (50) NULL, - RAW_RX_QUANTITY varchar(50) NULL, - RAW_RX_NDC varchar(50) NULL, + RAW_RX_QUANTITY varchar(50) NULL, + RAW_RX_NDC varchar(50) NULL, RAW_RXNORM_CUI varchar (50) NULL ) / @@ -126,6 +126,14 @@ CREATE TABLE SUPPLY ( MODIFIER_CD VARCHAR2(100) NOT NULL ) / +begin +PMN_DROPSQL('DROP TABLE prescribing_transfer'); +end; +/ +create global temporary table prescribing_transfer on commit preserve rows as +select * from prescribing where 1 = 0 +/ + create or replace procedure PCORNetPrescribing as begin @@ -203,33 +211,30 @@ commit; execute immediate 'create index supply_idx on supply (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('SUPPLY'); --- insert data with outer joins to ensure all records are included even if some data elements are missing -insert /*+ parallel(10) */ into prescribing ( - PATID - ,encounterid - ,RX_PROVIDERID - ,RX_ORDER_DATE -- using start_date from i2b2 - ,RX_ORDER_TIME -- using time start_date from i2b2 - ,RX_START_DATE - ,RX_END_DATE - ,RXNORM_CUI --using pcornet_med pcori_cui - new column! - ,RX_QUANTITY ---- modifier nval_num - ,RX_QUANTITY_UNIT - ,RX_REFILLS -- modifier nval_num - ,RX_DAYS_SUPPLY -- modifier nval_num - ,RX_FREQUENCY --modifier with basecode lookup - ,RX_BASIS --modifier with basecode lookup - ,RAW_RX_MED_NAME --- ,RAW_RX_FREQUENCY, - ,RAW_RXNORM_CUI +insert into prescribing_transfer ( + PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, + RX_QUANTITY, RX_QUANTITY_UNIT, RX_REFILLS, RX_DAYS_SUPPLY, RX_FREQUENCY, RX_BASIS, RAW_RX_MED_NAME, RAW_RXNORM_CUI ) -select /*+ parallel(10) */ distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH24:MI'), m.start_date start_date, m.end_date, mo.pcori_cui - ,quantity.nval_num quantity, 'NI' rx_quantity_unit, refills.nval_num refills, supply.nval_num supply, substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) frequency, - substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) basis - , substr(mo.c_name, 1, 50) raw_rx_med_name, substr(mo.c_basecode, 1, 50) raw_rxnorm_cui - from i2b2medfact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode -inner join encounter enc on enc.encounterid = m.encounter_Num --- TODO: This join adds several minutes to the load - must be debugged + select + m.patient_num PATID, + m.Encounter_Num ENCOUNTERID, + m.provider_id RX_PROVIDERID, + m.start_date RX_ORDER_DATE, + to_char(m.start_date,'HH24:MI') RX_ORDER_TIME, + m.start_date RX_START_DATE, + m.end_date RX_END_DATE, + mo.pcori_cui RXNORM_CUI, + quantity.nval_num RX_QUANTITY, + 'NI' RX_QUANTITY_UNIT, + refills.nval_num RX_REFILLS, + supply.nval_num RX_DAYS_SUPPLY, + substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) RX_FREQUENCY, + substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) RX_BASIS, + substr(mo.c_name, 1, 50) RAW_RX_MED_NAME, + substr(mo.c_basecode, 1, 50) RAW_RXNORM_CUI + + from i2b2medfact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode + inner join encounter enc on enc.encounterid = m.encounter_Num left join basis on m.encounter_num = basis.encounter_num @@ -266,7 +271,20 @@ inner join encounter enc on enc.encounterid = m.encounter_Num and m.provider_id = supply.provider_id and m.instance_num = supply.instance_num -where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); + where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); + +commit; + +insert into prescribing ( + PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, + RX_QUANTITY, RX_QUANTITY_UNIT, RX_REFILLS, RX_DAYS_SUPPLY, RX_FREQUENCY, RX_BASIS, RAW_RX_MED_NAME, RAW_RXNORM_CUI +) +select distinct + PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, + RX_QUANTITY, RX_QUANTITY_UNIT, RX_REFILLS, RX_DAYS_SUPPLY, RX_FREQUENCY, RX_BASIS, RAW_RX_MED_NAME, RAW_RXNORM_CUI +from prescribing_transfer; + +commit; execute immediate 'create index prescribing_idx on prescribing (PATID, ENCOUNTERID)'; GATHER_TABLE_STATS('PRESCRIBING'); @@ -277,7 +295,8 @@ BEGIN PCORNetPrescribing(); END; / +truncate table prescribing_transfer +/ insert into cdm_status (status, last_update, records) select 'prescribing', sysdate, count(*) from prescribing / -select 1 from cdm_status where status = 'prescribing' ---SELECT count(PRESCRIBINGID) from prescribing where rownum = 1 \ No newline at end of file +select 1 from cdm_status where status = 'prescribing' \ No newline at end of file From 80a12563c3c3210b373aa7bb37e93d713fef43cc Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 30 Mar 2018 15:10:23 -0500 Subject: [PATCH 341/507] Added nested loop hint --- Oracle/prescribing.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index bc62b16..24e1d87 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -211,7 +211,7 @@ commit; execute immediate 'create index supply_idx on supply (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('SUPPLY'); -insert into prescribing_transfer ( +insert /*+ use_nl(freq quantity supply refills) parallel(10) */ into prescribing_transfer ( PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, RX_QUANTITY, RX_QUANTITY_UNIT, RX_REFILLS, RX_DAYS_SUPPLY, RX_FREQUENCY, RX_BASIS, RAW_RX_MED_NAME, RAW_RXNORM_CUI ) @@ -275,7 +275,7 @@ insert into prescribing_transfer ( commit; -insert into prescribing ( +insert /*+ parallel(10) */ into prescribing ( PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, RX_QUANTITY, RX_QUANTITY_UNIT, RX_REFILLS, RX_DAYS_SUPPLY, RX_FREQUENCY, RX_BASIS, RAW_RX_MED_NAME, RAW_RXNORM_CUI ) From b5355eeaa7da9098fa8c67bb87b35b0a25fdbe93 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 30 Mar 2018 15:52:15 -0500 Subject: [PATCH 342/507] Testing insert into prescribing only --- Oracle/prescribing.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 24e1d87..2db879f 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -137,6 +137,7 @@ select * from prescribing where 1 = 0 create or replace procedure PCORNetPrescribing as begin +/* PMN_DROPSQL('drop index prescribing_idx'); PMN_DROPSQL('drop index basis_idx'); PMN_DROPSQL('drop index freq_idx'); @@ -210,12 +211,13 @@ commit; execute immediate 'create index supply_idx on supply (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('SUPPLY'); +*/ -insert /*+ use_nl(freq quantity supply refills) parallel(10) */ into prescribing_transfer ( +insert /*+ append */ into prescribing_transfer ( PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, RX_QUANTITY, RX_QUANTITY_UNIT, RX_REFILLS, RX_DAYS_SUPPLY, RX_FREQUENCY, RX_BASIS, RAW_RX_MED_NAME, RAW_RXNORM_CUI ) - select + select /*+ use_nl(freq quantity supply refills) parallel(10) */ m.patient_num PATID, m.Encounter_Num ENCOUNTERID, m.provider_id RX_PROVIDERID, From e3de6c2f3478dd6dcf6bfdb5989d67abe3d51151 Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 2 Apr 2018 09:46:53 -0500 Subject: [PATCH 343/507] Production version building all tables --- Oracle/prescribing.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 2db879f..42789ee 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -137,7 +137,6 @@ select * from prescribing where 1 = 0 create or replace procedure PCORNetPrescribing as begin -/* PMN_DROPSQL('drop index prescribing_idx'); PMN_DROPSQL('drop index basis_idx'); PMN_DROPSQL('drop index freq_idx'); @@ -211,7 +210,6 @@ commit; execute immediate 'create index supply_idx on supply (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('SUPPLY'); -*/ insert /*+ append */ into prescribing_transfer ( PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, @@ -277,7 +275,7 @@ insert /*+ append */ into prescribing_transfer ( commit; -insert /*+ parallel(10) */ into prescribing ( +insert /*+ append parallel(10) */ into prescribing ( PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, RX_QUANTITY, RX_QUANTITY_UNIT, RX_REFILLS, RX_DAYS_SUPPLY, RX_FREQUENCY, RX_BASIS, RAW_RX_MED_NAME, RAW_RXNORM_CUI ) From 911cea3377b5bb0095023d1092e1314407013798 Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 10 Apr 2018 09:11:41 -0500 Subject: [PATCH 344/507] Added csv loader --- Oracle/condition.sql | 3 -- Oracle/death.sql | 3 -- Oracle/death_cause.sql | 2 -- Oracle/demographic.sql | 2 -- Oracle/diagnosis.sql | 3 -- Oracle/dispensing.sql | 3 -- Oracle/encounter.sql | 3 -- Oracle/enrollment.sql | 2 -- Oracle/harvest.sql | 32 ----------------- Oracle/lab_result_cm.sql | 3 -- Oracle/med_admin.sql | 2 -- Oracle/pcornet_init.sql | 21 +++++++++++ Oracle/pcornet_loader.sql | 2 -- Oracle/pcornet_trial.sql | 2 -- Oracle/prescribing.sql | 3 -- Oracle/pro_cm.sql | 2 -- Oracle/procedures.sql | 3 -- Oracle/provider.sql | 6 ++-- Oracle/vital.sql | 3 -- csv_load.py | 69 +++++++++++++++++++++++++++++++++++ i2p_tasks.py | 75 +++++++++++++++++++++++++++++++++------ 21 files changed, 157 insertions(+), 87 deletions(-) create mode 100644 csv_load.py diff --git a/Oracle/condition.sql b/Oracle/condition.sql index de57c8f..eef156e 100644 --- a/Oracle/condition.sql +++ b/Oracle/condition.sql @@ -1,8 +1,5 @@ /** condition - create and populate the condition table. */ - -select encounterid from encounter where 'dep' = 'encounter.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE condition'); END; diff --git a/Oracle/death.sql b/Oracle/death.sql index 4eb3e5b..f7fccd5 100644 --- a/Oracle/death.sql +++ b/Oracle/death.sql @@ -1,8 +1,5 @@ /** death - create and populate the death table. */ - -select patid from demographic where 'dep' = 'demographic.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE death'); END; diff --git a/Oracle/death_cause.sql b/Oracle/death_cause.sql index 120608e..b6fda62 100644 --- a/Oracle/death_cause.sql +++ b/Oracle/death_cause.sql @@ -1,7 +1,5 @@ /** death_cause - create the death_cause table. */ -select synonym_name from all_synonyms where 'dep' = 'pcornet_init.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE death_cause'); END; diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql index 70d78f4..af0866b 100644 --- a/Oracle/demographic.sql +++ b/Oracle/demographic.sql @@ -1,7 +1,5 @@ /** demographic - create and populate the demographic table. */ -select synonym_name from all_synonyms where 'dep' = 'pcornet_init.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE demographic'); END; diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 71979be..8fd7a2f 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -1,8 +1,5 @@ /** diagnosis - create and populate the diagnosis table. */ - -select encounterid from encounter where 'dep' = 'encounter.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE diagnosis'); END; diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index 1331b63..094ea7e 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -1,8 +1,5 @@ /** dispensing - create and populate the dispensing table. */ - ---select encounterid from encounter where 'dep' = 'encounter.sql'; - BEGIN PMN_DROPSQL('DROP TABLE dispensing'); END; diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index b0f24e1..a009063 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -1,8 +1,5 @@ /** encounter - create and populate the encounter table. */ - -select patid from demographic where 'dep' = 'demographic.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE encounter'); END; diff --git a/Oracle/enrollment.sql b/Oracle/enrollment.sql index 7c5084f..b51c825 100644 --- a/Oracle/enrollment.sql +++ b/Oracle/enrollment.sql @@ -1,7 +1,5 @@ /** enrollment - create and populate the enrollment table. */ -select synonym_name from all_synonyms where 'dep' = 'pcornet_init.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE enrollment'); END; diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index ca0d590..6ccfed7 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -1,37 +1,5 @@ /** harvest - create and populate the harvest table. */ -select conditionid from condition where 'dep' = 'condition.sql' -/ -select patid from death where 'dep' = 'death.sql' -/ -select patid from death_cause where 'dep' = 'death_cause.sql' -/ -select diagnosisid from diagnosis where 'dep' = 'diagnosis.sql' -/ -select dispensingid from dispensing where 'dep' = 'dispensing.sql' -/ -select patid from enrollment where 'dep' = 'enrollment.sql' -/ -select lab_result_cm_id from lab_result_cm where 'dep' = 'lab_result_cm.sql' -/ -select medadminid from med_admin where 'dep' = 'med_admin.sql' -/ -select obsclinid from obs_clin where 'dep' = 'obs_clin.sql' -/ -select obsgenid from obs_gen where 'dep' = 'obs_gen.sql' -/ -select patid from pcornet_trial where 'dep' = 'pcornet_trial.sql' -/ -select prescribingid from prescribing where 'dep' = 'prescribing.sql' -/ -select pro_cm_id from pro_cm where 'dep' = 'pro_cm.sql' -/ -select proceduresid from procedures where 'dep' = 'procedures.sql' -/ -select providerid from provider where 'dep' = 'provider.sql' -/ -select vitalid from vital where 'dep' = 'vital.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE harvest'); END; diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 555560e..534f40e 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -1,8 +1,5 @@ /** lab_result_cm - create and populate the lab_result_cm table. */ - -select encounterid from encounter where 'dep' = 'encounter.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE lab_result_cm'); END; diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index bd03536..9f32f9a 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -1,7 +1,5 @@ /** med_admin - create and populate the med_admin table. */ -select synonym_name from all_synonyms where 'dep' = 'pcornet_init.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE med_admin'); END; diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 395fb1e..ffc9e74 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -1,5 +1,26 @@ /** pcornet_init - create helper functions and procedures */ +-- Make sure the ethnicity code has been added to the patient dimension +-- See update_ethnicity_pdim.sql +select ethnicity_cd from "&&i2b2_data_schema".patient_dimension where 1=0 +/ +-- Make sure that the providerid column has been added to the visit dimension +select providerid from "&&i2b2_data_schema".visit_dimension where 1=0 +/ +-- Make sure the inout_cd has been populated +-- See heron_encounter_style.sql +select case when qty = 0 then 1/0 else 1 end inout_cd_populated from ( + select count(*) qty from "&&i2b2_data_schema".visit_dimension where inout_cd is not null + ) +/ +-- Make sure the RXNorm mapping table exists +select rxcui from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id where 1=0 +/ +-- Make sure the observation fact medication table is populated +select case when qty > 0 then 1 else 1/0 end obs_fact_meds_populated from ( + select count(*) qty from observation_fact_meds + ) +/ create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS BEGIN DBMS_STATS.GATHER_TABLE_STATS ( diff --git a/Oracle/pcornet_loader.sql b/Oracle/pcornet_loader.sql index 213e2fa..48a901b 100644 --- a/Oracle/pcornet_loader.sql +++ b/Oracle/pcornet_loader.sql @@ -1,5 +1,3 @@ -select networkid from harvest where 'dep' = 'harvest.sql' -/ insert into cdm_status (status, last_update) values ('pcornet_loader', sysdate ) / select 1 from cdm_status where status = 'pcornet_loader' \ No newline at end of file diff --git a/Oracle/pcornet_trial.sql b/Oracle/pcornet_trial.sql index 6f35af5..f645aec 100644 --- a/Oracle/pcornet_trial.sql +++ b/Oracle/pcornet_trial.sql @@ -1,7 +1,5 @@ /** pcornet_trial - create the pcornet_trial table. */ -select synonym_name from all_synonyms where 'dep' = 'pcornet_init.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE pcornet_trial'); END; diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 42789ee..f8b7adf 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -1,8 +1,5 @@ /** prescribing - create and populate the prescribing table. */ - -select encounterid from encounter where 'dep' = 'encounter.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE prescribing'); END; diff --git a/Oracle/pro_cm.sql b/Oracle/pro_cm.sql index 03c1c57..1116076 100644 --- a/Oracle/pro_cm.sql +++ b/Oracle/pro_cm.sql @@ -1,7 +1,5 @@ /** pro_cm - create the pro_cm table. */ -select synonym_name from all_synonyms where 'dep' = 'pcornet_init.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE pro_cm'); END; diff --git a/Oracle/procedures.sql b/Oracle/procedures.sql index 3e47788..a3d69a8 100644 --- a/Oracle/procedures.sql +++ b/Oracle/procedures.sql @@ -1,8 +1,5 @@ /** procedures - create and populate the procedures table. */ - -select encounterid from encounter where 'dep' = 'encounter.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE procedures'); END; diff --git a/Oracle/provider.sql b/Oracle/provider.sql index 22a42e7..b486719 100644 --- a/Oracle/provider.sql +++ b/Oracle/provider.sql @@ -1,7 +1,5 @@ --------------------------------------------------------------------------------- --- PROVIDER --------------------------------------------------------------------------------- - +/** provider - create and populate the provider table. +*/ BEGIN PMN_DROPSQL('DROP TABLE provider'); END; diff --git a/Oracle/vital.sql b/Oracle/vital.sql index 5873cb9..ceda92a 100644 --- a/Oracle/vital.sql +++ b/Oracle/vital.sql @@ -1,8 +1,5 @@ /** vital - create and populate the vital table. */ - -select encounterid from encounter where 'dep' = 'encounter.sql' -/ BEGIN PMN_DROPSQL('DROP TABLE vital'); END; diff --git a/csv_load.py b/csv_load.py new file mode 100644 index 0000000..6382c10 --- /dev/null +++ b/csv_load.py @@ -0,0 +1,69 @@ +from collections import defaultdict +from csv import DictReader +from datetime import datetime +from etl_tasks import DBAccessTask +from param_val import StrParam, IntParam +from sqlalchemy import func, MetaData, Table, Column +from sqlalchemy.types import String + +import logging +import sqlalchemy as sqla + +log = logging.getLogger(__name__) + +class LoadCSV(DBAccessTask): + tablename = StrParam() + csvname = StrParam() + rowcount = IntParam(default=1) + + def complete(self) -> bool: + #return False + db = self._dbtarget().engine + table = Table(self.tablename, sqla.MetaData()) + if not table.exists(bind=db): + log.info('no such table: %s', self.tablename) + return False + with self.connection() as q: + actual = q.scalar('select records from cdm_status where status = \'%s\'' % self.tablename) + actual = 0 if actual == None else actual + log.info('table %s has %d rows', self.tablename, actual) + return actual >= self.rowcount # type: ignore # sqla + + def run(self) -> None: + self.load() + self.setStatus() + + def load(self) -> None: + def sz(l, chunk=16): + return max(chunk, chunk * ((l + chunk - 1) // chunk)) + + db = self._dbtarget().engine + schema = MetaData() + l = list() + + with open(self.csvname) as fin: + dr = DictReader(fin) + + mcl = defaultdict(int) + for row in dr: + l.append(row) + for col in dr.fieldnames: + mcl[col] = sz(max(mcl[col], len(row[col]))) + + columns = ([Column(n, String(mcl[n])) for n in dr.fieldnames]) + table = Table(self.tablename, schema, *columns) + if table.exists(bind=db): + table.drop(db) + table.create(db) + + db.execute(table.insert(), l) + + def setStatus(self) -> None: + statusTable = Table("cdm_status", MetaData(), Column('STATUS'), Column('LAST_UPDATE'), Column('RECORDS')) + + db = self._dbtarget().engine + + with self.connection() as q: + actual = q.scalar(sqla.select([func.count()]).select_from(self.tablename)) + + db.execute(statusTable.insert(), [{'STATUS':self.tablename, 'LAST_UPDATE':datetime.now(), 'RECORDS':actual}]) \ No newline at end of file diff --git a/i2p_tasks.py b/i2p_tasks.py index 28d053f..c21dc29 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -1,6 +1,7 @@ """i2p_tasks -- Luigi CDM task support. """ +from csv_load import LoadCSV from etl_tasks import SqlScriptTask from param_val import IntParam from script_lib import Script @@ -16,55 +17,84 @@ def variables(self) -> Environment: return dict(datamart_id='C4UK', datamart_name='University of Kansas', i2b2_data_schema='BLUEHERONDATA', min_pat_list_date_dd_mon_rrrr='01-Jan-2010', min_visit_date_dd_mon_rrrr='01-Jan-2010', i2b2_meta_schema='BLUEHERONMETADATA', enrollment_months_back='2', network_id='C4', - network_name='GPC') + network_name='GPC', i2b2_etl_schema='HERON_ETL_3') class condition(CDMScriptTask): script = Script.condition + def requires(self): + return [encounter()] class death(CDMScriptTask): script = Script.death + def requires(self): + return [demographic()] + class death_cause(CDMScriptTask): script = Script.death_cause + def requires(self): + return [pcornet_init()] + class demographic(CDMScriptTask): script = Script.demographic + def requires(self): + return [pcornet_init()] + class diagnosis(CDMScriptTask): script = Script.diagnosis + def requires(self): + return [encounter()] + class dispensing(CDMScriptTask): script = Script.dispensing + def requires(self): + return [encounter()] + class encounter(CDMScriptTask): script = Script.encounter + def requires(self): + return [demographic()] + class enrollment(CDMScriptTask): script = Script.enrollment + def requires(self): + return [pcornet_init()] + class harvest(CDMScriptTask): script = Script.harvest + def requires(self): + return [condition(), death(), death_cause(), diagnosis(), dispensing(), enrollment(), + lab_result_cm(), med_admin(), obs_clin(), obs_gen(), pcornet_trial(), + prescribing(), pro_cm(), procedures(), provider(), vital()] class lab_result_cm(CDMScriptTask): script = Script.lab_result_cm + def requires(self): + return [encounter()] + class med_admin(CDMScriptTask): script = Script.med_admin - -class med_admin_init(CDMScriptTask): - script = Script.med_admin_init + def requires(self): + return [pcornet_init()] class obs_clin(CDMScriptTask): @@ -80,12 +110,6 @@ class patient_chunks_survey(SqlScriptTask): patient_chunks = IntParam(default=200) patient_chunk_max = IntParam(default=None) - #def variables(self) -> Environment: - # return dict(chunk_qty=str(self.patient_chunks)) - - #def run(self) -> None: - # SqlScriptTask.run_bound(self, script_params=dict(chunk_qty=str(self.patient_chunks)) - def results(self) -> List[RowProxy]: with self.connection(event='survey results') as lc: q = ''' @@ -110,26 +134,44 @@ def results(self) -> List[RowProxy]: class pcornet_init(CDMScriptTask): script = Script.pcornet_init + def requires(self): + return [loadLabNormal(), loadHarvestLocal()] + class pcornet_loader(CDMScriptTask): script = Script.pcornet_loader + def requires(self): + return [harvest()] + class pcornet_trial(CDMScriptTask): script = Script.pcornet_trial + def requires(self): + return [pcornet_init()] + class prescribing(CDMScriptTask): script = Script.prescribing + def requires(self): + return [encounter()] + class pro_cm(CDMScriptTask): script = Script.pro_cm + def requires(self): + return [pcornet_init()] + class procedures(CDMScriptTask): script = Script.procedures + def requires(self): + return [encounter()] + class provider(CDMScriptTask): script = Script.provider @@ -137,3 +179,16 @@ class provider(CDMScriptTask): class vital(CDMScriptTask): script = Script.vital + + def requires(self): + return [encounter()] + + +class loadLabNormal(LoadCSV): + tablename = 'LABNORMAL' + csvname = 'labnormal.csv' + + +class loadHarvestLocal(LoadCSV): + tablename = 'HARVET_LOCAL' + csvname = 'harvest_local.csv' From cb77d35972725b06dd6ada5c4a6f9db59ae30a9c Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 10 Apr 2018 09:31:27 -0500 Subject: [PATCH 345/507] Added path to csv name --- i2p_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index c21dc29..c3edefc 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -186,9 +186,9 @@ def requires(self): class loadLabNormal(LoadCSV): tablename = 'LABNORMAL' - csvname = 'labnormal.csv' + csvname = 'Oracle/labnormal.csv' class loadHarvestLocal(LoadCSV): tablename = 'HARVET_LOCAL' - csvname = 'harvest_local.csv' + csvname = 'Oracle/harvest_local.csv' From e42709e2c0467a29a601f952cac2c71823ea7dc2 Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 10 Apr 2018 10:04:01 -0500 Subject: [PATCH 346/507] Use cdm_status table to complete harvest --- Oracle/harvest.sql | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index 6ccfed7..359282e 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -74,24 +74,24 @@ INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, hl.DEATH_DATE_MGMT, hl.MEDADMIN_START_DATE_MGMT, hl.MEDADMIN_END_DATE_MGMT, hl.OBSCLIN_DATE_MGMT, hl.OBSGEN_DATE_MGMT, - case when (select count(*) from demographic) > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, - case when (select count(*) from enrollment) > 0 then current_date else null end REFRESH_ENROLLMENT_DATE, - case when (select count(*) from encounter) > 0 then current_date else null end REFRESH_ENCOUNTER_DATE, - case when (select count(*) from diagnosis) > 0 then current_date else null end REFRESH_DIAGNOSIS_DATE, - case when (select count(*) from procedures) > 0 then current_date else null end REFRESH_PROCEDURES_DATE, - case when (select count(*) from vital) > 0 then current_date else null end REFRESH_VITAL_DATE, - case when (select count(*) from dispensing) > 0 then current_date else null end REFRESH_DISPENSING_DATE, - case when (select count(*) from lab_result_cm) > 0 then current_date else null end REFRESH_LAB_RESULT_CM_DATE, - case when (select count(*) from condition) > 0 then current_date else null end REFRESH_CONDITION_DATE, - case when (select count(*) from pro_cm) > 0 then current_date else null end REFRESH_PRO_CM_DATE, - case when (select count(*) from prescribing) > 0 then current_date else null end REFRESH_PRESCRIBING_DATE, - case when (select count(*) from pcornet_trial) > 0 then current_date else null end REFRESH_PCORNET_TRIAL_DATE, - case when (select count(*) from death) > 0 then current_date else null end REFRESH_DEATH_DATE, - case when (select count(*) from death_cause) > 0 then current_date else null end REFRESH_DEATH_CAUSE_DATE, - case when (select count(*) from med_admin) > 0 then current_date else null end REFRESH_MED_ADMIN_DATE, - case when (select count(*) from obs_clin) > 0 then current_date else null end REFRESH_OBS_CLIN_DATE, - case when (select count(*) from provider) > 0 then current_date else null end REFRESH_PROVIDER_DATE, - case when (select count(*) from obs_gen) > 0 then current_date else null end REFRESH_OBS_GEN_DATE + case when (select records from cdm_status where status = 'demographic') > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, + case when (select records from cdm_status where status = 'enrollment') > 0 then current_date else null end REFRESH_ENROLLMENT_DATE, + case when (select records from cdm_status where status = 'encounter') > 0 then current_date else null end REFRESH_ENCOUNTER_DATE, + case when (select records from cdm_status where status = 'diagnosis_finalize') > 0 then current_date else null end REFRESH_DIAGNOSIS_DATE, + case when (select records from cdm_status where status = 'procedures') > 0 then current_date else null end REFRESH_PROCEDURES_DATE, + case when (select records from cdm_status where status = 'vital') > 0 then current_date else null end REFRESH_VITAL_DATE, + case when (select records from cdm_status where status = 'dispensing_finalize') > 0 then current_date else null end REFRESH_DISPENSING_DATE, + case when (select records from cdm_status where status = 'lab_result_cm') > 0 then current_date else null end REFRESH_LAB_RESULT_CM_DATE, + case when (select records from cdm_status where status = 'condition_finalize') > 0 then current_date else null end REFRESH_CONDITION_DATE, + case when (select records from cdm_status where status = 'pro_cm') > 0 then current_date else null end REFRESH_PRO_CM_DATE, + case when (select records from cdm_status where status = 'prescribing') > 0 then current_date else null end REFRESH_PRESCRIBING_DATE, + case when (select records from cdm_status where status = 'pcornet_trial') > 0 then current_date else null end REFRESH_PCORNET_TRIAL_DATE, + case when (select records from cdm_status where status = 'death_finalize') > 0 then current_date else null end REFRESH_DEATH_DATE, + case when (select records from cdm_status where status = 'death_cause') > 0 then current_date else null end REFRESH_DEATH_CAUSE_DATE, + case when (select records from cdm_status where status = 'med_admin') > 0 then current_date else null end REFRESH_MED_ADMIN_DATE, + case when (select records from cdm_status where status = 'obs_clin') > 0 then current_date else null end REFRESH_OBS_CLIN_DATE, + case when (select records from cdm_status where status = 'provider') > 0 then current_date else null end REFRESH_PROVIDER_DATE, + case when (select records from cdm_status where status = 'obs_gen') > 0 then current_date else null end REFRESH_OBS_GEN_DATE from harvest_local hl; @@ -104,4 +104,3 @@ END; insert into cdm_status (status, last_update, records) select 'harvest', sysdate, count(*) from harvest / select 1 from cdm_status where status = 'harvest' ---SELECT count(NETWORKID) from Harvest where rownum = 1 \ No newline at end of file From 4c9ad5e7a027907c0bd8738fd06d8ae20d06fe53 Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 10 Apr 2018 10:11:10 -0500 Subject: [PATCH 347/507] Correcting typo in HARVEST_LOCAL --- i2p_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index c3edefc..764b3c4 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -190,5 +190,5 @@ class loadLabNormal(LoadCSV): class loadHarvestLocal(LoadCSV): - tablename = 'HARVET_LOCAL' + tablename = 'HARVEST_LOCAL' csvname = 'Oracle/harvest_local.csv' From e72e20b8d2019212342d9d262325f65d3a3065f9 Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 10 Apr 2018 11:34:31 -0500 Subject: [PATCH 348/507] Correcting typo in DEATH_DATE_MGMT --- Oracle/harvest_local.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/harvest_local.csv b/Oracle/harvest_local.csv index e428cb6..8805952 100644 --- a/Oracle/harvest_local.csv +++ b/Oracle/harvest_local.csv @@ -1,2 +1,2 @@ -"DATAMART_CLAIMS","DATAMART_EHR","BIRTH_DATE_MGMT","ENR_START_DATE_MGMT","ENR_END_DATE_MGMT","ADMIT_DATE_MGMT","DISCHARGE_DATE_MGMT","PX_DATE_MGMT","RX_ORDER_DATE_MGMT","RX_START_DATE_MGMT","RX_END_DATE_MGMT","DISPENSE_DATE_MGMT","LAB_ORDER_DATE_MGMT","SPECIMEN_DATE_MGMT","RESULT_DATE_MGMT","MEASURE_DATE_MGMT","ONSET_DATE_MGMT","REPORT_DATE_MGMT","RESOLVE_DATE_MGMT","PRO_DATE_MGMT","DEATH_DATE_MGTM","MEDADMIN_START_DATE_MGMT","MEDADMIN_END_DATE_MGMT","OBSCLIN_DATE_MGMT","OBSGEN_DATE_MGMT" +"DATAMART_CLAIMS","DATAMART_EHR","BIRTH_DATE_MGMT","ENR_START_DATE_MGMT","ENR_END_DATE_MGMT","ADMIT_DATE_MGMT","DISCHARGE_DATE_MGMT","PX_DATE_MGMT","RX_ORDER_DATE_MGMT","RX_START_DATE_MGMT","RX_END_DATE_MGMT","DISPENSE_DATE_MGMT","LAB_ORDER_DATE_MGMT","SPECIMEN_DATE_MGMT","RESULT_DATE_MGMT","MEASURE_DATE_MGMT","ONSET_DATE_MGMT","REPORT_DATE_MGMT","RESOLVE_DATE_MGMT","PRO_DATE_MGMT","DEATH_DATE_MGMT","MEDADMIN_START_DATE_MGMT","MEDADMIN_END_DATE_MGMT","OBSCLIN_DATE_MGMT","OBSGEN_DATE_MGMT" "01","02","03","03","03","03","03","03","03","03","04","03","03","03","03","03","03","03","03","03","03","03","03","03","03" From 964cabf66ecef781c719beb3955c19d7d5c890c7 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 11 Apr 2018 13:36:43 -0500 Subject: [PATCH 349/507] Fix for ticket 5197. --- Oracle/procedures.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Oracle/procedures.sql b/Oracle/procedures.sql index a3d69a8..d315c83 100644 --- a/Oracle/procedures.sql +++ b/Oracle/procedures.sql @@ -39,10 +39,11 @@ PMN_DROPSQL('drop index procedures_idx'); execute immediate 'truncate table procedures'; -insert into procedures( - patid, encounterid, enc_type, admit_date, px_date, providerid, px, px_type, px_source) +insert into procedures(patid, encounterid, enc_type, admit_date, px_date, providerid, px, px_type, px_source) select distinct fact.patient_num, enc.encounterid, enc.enc_type, enc.admit_date, fact.start_date, - fact.provider_id, SUBSTR(pr.pcori_basecode,INSTR(pr.pcori_basecode, ':')+1,11) px, SUBSTR(pr.c_fullname,18,2) pxtype, + fact.provider_id, SUBSTR(pr.pcori_basecode,INSTR(pr.pcori_basecode, ':')+1,11) px, + -- Decode can be eliminated if pcornet_proc is updated. + decode(SUBSTR(pr.c_fullname,18,2), 'HC', 'CH', SUBSTR(pr.c_fullname,18,2)) pxtype, -- All are billing for now - see https://informatics.gpcnetwork.org/trac/Project/ticket/491 'BI' px_source from i2b2fact fact From 3e678735491a1de9ac7c910219102945a8416073 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 11 Apr 2018 13:37:33 -0500 Subject: [PATCH 350/507] Fix for ticket 5085 --- Oracle/death.sql | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/Oracle/death.sql b/Oracle/death.sql index f7fccd5..757b150 100644 --- a/Oracle/death.sql +++ b/Oracle/death.sql @@ -17,29 +17,25 @@ begin execute immediate 'truncate table death'; -insert into death( patid, death_date, death_date_impute, death_source, death_match_confidence) -select distinct pat.patient_num, pat.death_date, -case when vital_status_cd like 'X%' then 'B' - when vital_status_cd like 'M%' then 'D' - when vital_status_cd like 'Y%' then 'N' - else 'OT' - end death_date_impute, - 'NI' death_source, - 'NI' death_match_confidence -from ( - /* KUMC specific fix to address unknown death dates */ - select - ibp.patient_num, - case when ibf.concept_cd is not null then DATE '2100-12-31' -- in accordance with the CDM v3 spec - else ibp.death_date end death_date, - case when ibf.concept_cd is not null then 'OT' - else upper(ibp.vital_status_cd) end vital_status_cd - from i2b2patient ibp - left join i2b2fact ibf - on ibp.patient_num=ibf.patient_num - and ibf.CONCEPT_CD='DEM|VITAL:yu' -) pat -where (pat.death_date is not null or vital_status_cd like 'Z%') and pat.patient_num in (select patid from demographic); +insert into death(patid, death_date, death_date_impute, death_source, death_match_confidence) +with death_source_map as ( + select 'DEM|VITAL:y' concept_cd, 'L' death_source, null unknown from dual + union all + select 'DEM|VITAL:yu' concept_cd, 'L' death_source, 1 unknown from dual + union all + select 'DEM|VITAL|SSA:y' concept_cd, 'D' death_source, null unknown from dual + union all + select 'NAACCR|1760:0' concept_cd, 'T' death_source, null unknown from dual + union all + select 'NAACCR|1760:4' concept_cd, 'T' death_source, null unknown from dual +) +select /*+ parallel(0) */ obs.patient_num patid + , case when dmap.unknown = 1 then DATE '2100-12-31' else obs.start_date end death_date + , case when dmap.unknown = 1 then 'OT' else 'N' end death_date_impute + , dmap.death_source + , 'NI' death_match_confidence +from pcornet_cdm.i2b2fact obs +join death_source_map dmap on obs.concept_cd = dmap.concept_cd; end; / From 706f2d8ba2a8f9b0f2237220526dffd75815c31a Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 12 Apr 2018 12:56:11 -0500 Subject: [PATCH 351/507] Added CDM 4 tables to backup configuration --- ...mon-Data-Model-v3dot0-parseable-fields.csv | 456 +++++++++--------- 1 file changed, 230 insertions(+), 226 deletions(-) diff --git a/2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv b/2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv index bf80f40..6b0fc84 100644 --- a/2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv +++ b/2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv @@ -1,11 +1,11 @@ -TABLE_NAME,FIELD_NAME,RDBMS_DATA_TYPE,SAS_DATA_TYPE,DATA_FORMAT,REPLICATED,UNIT_OF_MEASURE,VALUE_SET,VALUE_DESCRIPTION,DEFINITION,CDM_ORDER +TABLE_NAME,FIELD_NAME,RDBMS_DATA_TYPE,SAS_DATA_TYPE,DATA_FORMAT,REPLICATED,UNIT_OF_MEASURE,VALUE_SET,VALUE_DESCRIPTION,DEFINITION,CDM_ORDER DEMOGRAPHIC,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary person-level identifier used to link across tables. PATID is a pseudoidentifier with a consistent crosswalk to the true identifier retained by the source Data Partner. For analytical data sets requiring patient-level data, only the pseudoidentifier is used to link across all information belonging to a patient. -The PATID must be unique within the data source being queried. Creating a unique identifier within a CDRN would be beneficial and acceptable. The PATID is not the basis for linkages across partners.",1 -DEMOGRAPHIC,BIRTH_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date of birth.,2 -DEMOGRAPHIC,BIRTH_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Time of birth.,3 +The PATID must be unique within the data source being queried. Creating a unique identifier within a CDRN would be beneficial and acceptable. The PATID is not the basis for linkages across partners.",1 +DEMOGRAPHIC,BIRTH_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date of birth.,2 +DEMOGRAPHIC,BIRTH_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Time of birth.,3 DEMOGRAPHIC,SEX,RDBMS Text(2),SAS Char(2),,NO,,A;F;M;NI;UN;OT,"A = Ambiguous F = Female M = Male @@ -13,7 +13,7 @@ NI = No information UN = Unknown OT = Other","Administrative sex. -v2.0 guidance added: The ?Ambiguous? category may be used for individuals who are physically undifferentiated from birth. The ?Other? category may be used for individuals who are undergoing gender re-assignment.",4 +v2.0 guidance added: The ?Ambiguous? category may be used for individuals who are physically undifferentiated from birth. The ?Other? category may be used for individuals who are undergoing gender re-assignment.",4 DEMOGRAPHIC,HISPANIC,RDBMS Text(2),SAS Char(2),,NO,,Y;N;R;NI;UN;OT,"Y = Yes N = No R = Refuse to answer @@ -21,7 +21,7 @@ NI = No information UN = Unknown OT = Other","A person of Cuban, Mexican, Puerto Rican, South or Central American, or other Spanish culture or origin, regardless of race. -v2.0 amendment: The new categorical value of ?Refuse to answer? has been added.",5 +v2.0 amendment: The new categorical value of ?Refuse to answer? has been added.",5 DEMOGRAPHIC,RACE,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;05;06;07;NI;UN;OT,"01 = American Indian or Alaska Native 02 = Asian 03 = Black or African American @@ -42,23 +42,23 @@ Black or African American: A person having origins in any of the black racial gr Native Hawaiian or Other Pacific Islander: A person having origins in any of the original peoples of Hawaii, Guam, Samoa, or other Pacific Islands. -White: A person having origins in any of the original peoples of Europe, the Middle East, or North Africa.",6 +White: A person having origins in any of the original peoples of Europe, the Middle East, or North Africa.",6 DEMOGRAPHIC,BIOBANK_FLAG,RDBMS Text(1),SAS Char(1),,NO,,Y;N,"Y = Yes N = No","Flag to indicate that one or more biobanked specimens are stored and available for research use. Examples of biospecimens could include plasma, urine, or tissue. If biospecimens are available, locally maintained ?mapping tables? would be necessary to map between the DEMOGRAPHIC record and the originating biobanking system(s). -If no known biobanked specimens are available, this field should be marked ?No?.",7 -DEMOGRAPHIC,RAW_SEX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",8 -DEMOGRAPHIC,RAW_HISPANIC,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",9 -DEMOGRAPHIC,RAW_RACE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",10 -ENROLLMENT,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables.,11 +If no known biobanked specimens are available, this field should be marked ?No?.",7 +DEMOGRAPHIC,RAW_SEX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",8 +DEMOGRAPHIC,RAW_HISPANIC,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",9 +DEMOGRAPHIC,RAW_RACE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",10 +ENROLLMENT,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables.,11 ENROLLMENT,ENR_START_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date of the beginning of the enrollment period. If the exact date is unknown, use the first day of the month. -For implementation of the CDM, a long span of longitudinal data is desirable; however, especially for historical data more than a decade old, the appropriate beginning date should be determined by the partner?s knowledge of the validity and usability of the data. More specific guidance can be provided through implementation discussions.",12 -ENROLLMENT,ENR_END_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date of the end of the enrollment period. If the exact date is unknown, use the last day of the month.",13 +For implementation of the CDM, a long span of longitudinal data is desirable; however, especially for historical data more than a decade old, the appropriate beginning date should be determined by the partner?s knowledge of the validity and usability of the data. More specific guidance can be provided through implementation discussions.",12 +ENROLLMENT,ENR_END_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date of the end of the enrollment period. If the exact date is unknown, use the last day of the month.",13 ENROLLMENT,CHART,RDBMS Text(1),SAS Char(1),,NO,,Y;N,"Y = Yes N = No","Chart abstraction flag is intended to answer the question, ""Are you able to request (or review) charts for this person?"" This flag does not address chart availability. Mark as ""Yes"" if there are no contractual or other restrictions between you and the individual (or sponsor) that would prohibit you from requesting any chart for this member. -Note: This field is most relevant for health insurers that can request charts from affiliated providers. This field allows exclusion of patients from studies that require chart review to validate exposures and/or outcomes. It identifies patients for whom charts are never available and for whom the chart can never be requested.",14 +Note: This field is most relevant for health insurers that can request charts from affiliated providers. This field allows exclusion of patients from studies that require chart review to validate exposures and/or outcomes. It identifies patients for whom charts are never available and for whom the chart can never be requested.",14 ENROLLMENT,ENR_BASIS,RDBMS Text(1),SAS Char(1),,NO,,I;G;A;E,"I = Insurance G = Geography A = Algorithmic @@ -73,15 +73,15 @@ Geography: An assertion of complete data capture between the start and end dates Algorithmic: An assertion of complete data capture between the start and end dates, based on a locally developed or applied algorithm, often using multiple criteria -Encounter-based: The start and stop dates are populated from the earliest-observed encounter and latest-observed encounter.",15 -ENCOUNTER,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier. Used to link across tables, including the ENCOUNTER, DIAGNOSIS, and PROCEDURE tables.",16 -ENCOUNTER,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables.,17 -ENCOUNTER,ADMIT_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Encounter or admission date.,18 -ENCOUNTER,ADMIT_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Encounter or admission time.,19 -ENCOUNTER,DISCHARGE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Discharge date. Should be populated for all Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),20 -ENCOUNTER,DISCHARGE_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Discharge time.,21 -ENCOUNTER,PROVIDERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Provider code for the provider who is most responsible for this encounter. For encounters with multiple providers choose one so the encounter can be linked to the diagnosis and procedure tables. As with the PATID, the provider code is a pseudoidentifier with a consistent crosswalk to the real identifier.",22 -ENCOUNTER,FACILITY_LOCATION,RDBMS Text(3),SAS Char(3),,NO,,,,Geographic location (3 digit zip code). Should be null if not recorded in source system (modification made from ?blank? to ?null? in v3.0).,23 +Encounter-based: The start and stop dates are populated from the earliest-observed encounter and latest-observed encounter.",15 +ENCOUNTER,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier. Used to link across tables, including the ENCOUNTER, DIAGNOSIS, and PROCEDURE tables.",16 +ENCOUNTER,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables.,17 +ENCOUNTER,ADMIT_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Encounter or admission date.,18 +ENCOUNTER,ADMIT_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Encounter or admission time.,19 +ENCOUNTER,DISCHARGE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Discharge date. Should be populated for all Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),20 +ENCOUNTER,DISCHARGE_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Discharge time.,21 +ENCOUNTER,PROVIDERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Provider code for the provider who is most responsible for this encounter. For encounters with multiple providers choose one so the encounter can be linked to the diagnosis and procedure tables. As with the PATID, the provider code is a pseudoidentifier with a consistent crosswalk to the real identifier.",22 +ENCOUNTER,FACILITY_LOCATION,RDBMS Text(3),SAS Char(3),,NO,,,,Geographic location (3 digit zip code). Should be null if not recorded in source system (modification made from ?blank? to ?null? in v3.0).,23 ENCOUNTER,ENC_TYPE,RDBMS Text(2),SAS Char(2),,NO,,AV;ED;EI;IP;IS;OA;NI;UN;OT,"AV = Ambulatory Visit ED = Emergency Department EI = Emergency Department Admit to Inpatient Hospital Stay (permissible substitution) @@ -103,15 +103,15 @@ Inpatient Hospital Stay: Includes all inpatient stays, including: same-day hospi Non-Acute Institutional Stay: Includes hospice, skilled nursing facility (SNF), rehab center, nursing home, residential, overnight non-hospital dialysis and other non-hospital stays. -Other Ambulatory Visit: Includes other non-overnight AV encounters such as hospice visits, home health visits, skilled nursing facility visits, other non-hospital visits, as well as telemedicine, telephone and email consultations. May also include ""lab only"" visits (when a lab is ordered outside of a patient visit), ""pharmacy only"" (e.g., when a patient has a refill ordered without a face-to-face visit), ""imaging only"", etc.",24 +Other Ambulatory Visit: Includes other non-overnight AV encounters such as hospice visits, home health visits, skilled nursing facility visits, other non-hospital visits, as well as telemedicine, telephone and email consultations. May also include ""lab only"" visits (when a lab is ordered outside of a patient visit), ""pharmacy only"" (e.g., when a patient has a refill ordered without a face-to-face visit), ""imaging only"", etc.",24 ENCOUNTER,FACILITYID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary local facility code that identifies the hospital or clinic. Used for chart abstraction and validation. -FACILITYID can be a true identifier, or a pseudoidentifier with a consistent crosswalk to the true identifier retained by the source Data Partner.",25 +FACILITYID can be a true identifier, or a pseudoidentifier with a consistent crosswalk to the true identifier retained by the source Data Partner.",25 ENCOUNTER,DISCHARGE_DISPOSITION,RDBMS Text(2),SAS Char(2),,NO,,A;E;NI;UN;OT,"A = Discharged alive E = Expired NI = No information UN = Unknown -OT = Other",Vital status at discharge. Should be populated for Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),26 +OT = Other",Vital status at discharge. Should be populated for Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),26 ENCOUNTER,DISCHARGE_STATUS,RDBMS Text(2),SAS Char(2),,NO,,AF;AL;AM;AW;EX;HH;HO;HS;IP;NH;RH;RS;SH;SN;NI;UN;OT,"AF = Adult Foster Home AL = Assisted Living Facility AM = Against Medical Advice @@ -128,16 +128,16 @@ SH = Still In Hospital SN = Skilled Nursing Facility NI = No information UN = Unknown -OT = Other",Discharge status. Should be populated for Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),27 +OT = Other",Discharge status. Should be populated for Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),27 ENCOUNTER,DRG,RDBMS Text(3),SAS Char(3),,NO,,,,"3-digit Diagnosis Related Group (DRG). Should be populated for IP and IS encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for AV or OA encounters. Use leading zeroes for codes less than 100. (Additional guidance added in v3.0 for the EI encounter type.) The DRG is used for reimbursement for inpatient encounters. It is a Medicare requirement that combines diagnoses into clinical concepts for billing. Frequently used in observational data analyses. -",28 +",28 ENCOUNTER,DRG_TYPE,RDBMS Text(2),SAS Char(2),,NO,,01;02;NI;UN;OT,"01 = CMS-DRG (old system) 02 = MS-DRG (current system) NI = No information UN = Unknown -OT = Other",DRG code version. MS-DRG (current system) began on 10/1/2007. Should be populated for IP and IS encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for AV or OA encounters. (Additional guidance added in v3.0 for the EI encounter type.),29 +OT = Other",DRG code version. MS-DRG (current system) began on 10/1/2007. Should be populated for IP and IS encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for AV or OA encounters. (Additional guidance added in v3.0 for the EI encounter type.),29 ENCOUNTER,ADMITTING_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,AF;AL;AV;ED;HH;HO;HS;IP;NH;RH;RS;SN;NI;UN;OT,"AF = Adult Foster Home AL = Assisted Living Facility AV = Ambulatory Visit @@ -152,20 +152,20 @@ RS = Residential Facility SN = Skilled Nursing Facility NI = No information UN = Unknown -OT = Other",Admitting source. Should be populated for Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),30 +OT = Other",Admitting source. Should be populated for Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),30 ENCOUNTER,RAW_SITEID,RDBMS Text(x),SAS Char(x),,NO,,,,"This field is new to v2.0. Optional field for locally-defined identifier intended for local use; for example, where a network may have multiple sites contributing to a central data repository. -This attribute may be sensitive in certain contexts; the intent is for internal network use only, and not to enable site quality comparisons.",31 -ENCOUNTER,RAW_ENC_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",32 -ENCOUNTER,RAW_DISCHARGE_DISPOSITION,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",33 -ENCOUNTER,RAW_DISCHARGE_STATUS,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",34 -ENCOUNTER,RAW_DRG_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",35 -ENCOUNTER,RAW_ADMITTING_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",36 -DIAGNOSIS,DIAGNOSISID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",37 -DIAGNOSIS,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,38 -DIAGNOSIS,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. Used to link across tables.,39 +This attribute may be sensitive in certain contexts; the intent is for internal network use only, and not to enable site quality comparisons.",31 +ENCOUNTER,RAW_ENC_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",32 +ENCOUNTER,RAW_DISCHARGE_DISPOSITION,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",33 +ENCOUNTER,RAW_DISCHARGE_STATUS,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",34 +ENCOUNTER,RAW_DRG_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",35 +ENCOUNTER,RAW_ADMITTING_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",36 +DIAGNOSIS,DIAGNOSISID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",37 +DIAGNOSIS,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,38 +DIAGNOSIS,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. Used to link across tables.,39 DIAGNOSIS,ENC_TYPE,RDBMS Text(2),SAS Char(2),,YES,,AV;ED;EI;IP;IS;OA;NI;UN;OT,"AV = Ambulatory Visit ED = Emergency Department EI=Emergency Department Admit to Inpatient Hospital Stay (permissible substitution) @@ -174,12 +174,12 @@ IS = Non-Acute Institutional Stay OA = Other Ambulatory Visit NI = No information UN = Unknown -OT = Other",Please note: This is a field replicated from the ENCOUNTER table. See the ENCOUNTER table for definitions. ,40 -DIAGNOSIS,ADMIT_DATE,RDBMS Date,SAS Date (Numeric),,YES,DATE,,,Please note: This is a field replicated from the ENCOUNTER table. See the ENCOUNTER table for definitions. ,41 -DIAGNOSIS,PROVIDERID,RDBMS Text(x),SAS Char(x),,YES,,,,Please note: This is a field replicated from the ENCOUNTER table. See the ENCOUNTER table for definitions. ,42 +OT = Other",Please note: This is a field replicated from the ENCOUNTER table. See the ENCOUNTER table for definitions. ,40 +DIAGNOSIS,ADMIT_DATE,RDBMS Date,SAS Date (Numeric),,YES,DATE,,,Please note: This is a field replicated from the ENCOUNTER table. See the ENCOUNTER table for definitions. ,41 +DIAGNOSIS,PROVIDERID,RDBMS Text(x),SAS Char(x),,YES,,,,Please note: This is a field replicated from the ENCOUNTER table. See the ENCOUNTER table for definitions. ,42 DIAGNOSIS,DX,RDBMS Text(18),SAS Char(18),,NO,,,,"Diagnosis code. -Leading zeroes and different levels of decimal precision are permissible in this field. Please populate the exact textual value of this diagnosis code, but remove source-specific suffixes and prefixes. Other codes should be listed as recorded in the source data.",43 +Leading zeroes and different levels of decimal precision are permissible in this field. Please populate the exact textual value of this diagnosis code, but remove source-specific suffixes and prefixes. Other codes should be listed as recorded in the source data.",43 DIAGNOSIS,DX_TYPE,RDBMS Text(2),SAS Char(2),,NO,,09;10;11;SM;NI;UN;OT,"09 = ICD-9-CM 10 = ICD-10-CM 11 = ICD-11-CM @@ -190,7 +190,7 @@ OT = Other","Diagnosis code type. We provide values for ICD and SNOMED code types. Other code types will be added as new terminologies are more widely used. -Please note: The ?Other? category is meant to identify internal use ontologies and codes.",44 +Please note: The ?Other? category is meant to identify internal use ontologies and codes.",44 DIAGNOSIS,DX_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,AD;DI;FI;IN;NI;UN;OT,"AD = Admitting DI = Discharge FI = Final @@ -199,7 +199,7 @@ NI = No information UN = Unknown OT = Other","Classification of diagnosis source. We include these categories to allow some flexibility in implementation. The context is to capture available diagnoses recorded during a specific encounter. It is not necessary to populate interim diagnoses unless readily available. -Ambulatory encounters would generally be expected to have a source of ?Final.?",45 +Ambulatory encounters would generally be expected to have a source of ?Final.?",45 DIAGNOSIS,PDX,RDBMS Text(2),SAS Char(2),,NO,,P;S;X;NI;UN;OT,"P = Principal S = Secondary X = Unable to Classify @@ -209,14 +209,14 @@ OT = Other","Principal discharge diagnosis flag. Relevant only on IP and IS enco For ED, AV, and OA encounter types, mark as X = Unable to Classify. (Billing systems do not require a primary diagnosis for ambulatory visits (eg, professional services)) -One principal diagnosis per encounter is expected, although in some instances more than one diagnosis may be flagged as principal. (Modification made in v3.0 to specify ?per encounter.?)",46 -DIAGNOSIS,RAW_DX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",47 -DIAGNOSIS,RAW_DX_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",48 -DIAGNOSIS,RAW_DX_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",49 -DIAGNOSIS,RAW_PDX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",50 -PROCEDURES,PROCEDURESID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",51 -PROCEDURES,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,52 -PROCEDURES,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. Used to link across tables.,53 +One principal diagnosis per encounter is expected, although in some instances more than one diagnosis may be flagged as principal. (Modification made in v3.0 to specify ?per encounter.?)",46 +DIAGNOSIS,RAW_DX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",47 +DIAGNOSIS,RAW_DX_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",48 +DIAGNOSIS,RAW_DX_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",49 +DIAGNOSIS,RAW_PDX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",50 +PROCEDURES,PROCEDURESID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",51 +PROCEDURES,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,52 +PROCEDURES,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. Used to link across tables.,53 PROCEDURES,ENC_TYPE,RDBMS Text(2),SAS Char(2),,YES,,AV;ED;EI;IP;IS;OA;NI;UN;OT,"AV = Ambulatory Visit ED = Emergency Department EI=Emergency Department Admit to Inpatient Hospital Stay (permissible substitution) @@ -225,13 +225,13 @@ IS = Non-Acute Institutional Stay OA = Other Ambulatory Visit NI = No information UN = Unknown -OT = Other",Please note: This is a field replicated from the ENCOUNTER table. See ENCOUNTER table for definitions.,54 -PROCEDURES,ADMIT_DATE,RDBMS Date,SAS Date (Numeric),,YES,DATE,,,Please note: This is a field replicated from the ENCOUNTER table. See ENCOUNTER table for definitions.,55 -PROCEDURES,PROVIDERID,RDBMS Text(x),SAS Char(x),,YES,,,,Please note: This is a field replicated from the ENCOUNTER table. See ENCOUNTER table for definitions.,56 +OT = Other",Please note: This is a field replicated from the ENCOUNTER table. See ENCOUNTER table for definitions.,54 +PROCEDURES,ADMIT_DATE,RDBMS Date,SAS Date (Numeric),,YES,DATE,,,Please note: This is a field replicated from the ENCOUNTER table. See ENCOUNTER table for definitions.,55 +PROCEDURES,PROVIDERID,RDBMS Text(x),SAS Char(x),,YES,,,,Please note: This is a field replicated from the ENCOUNTER table. See ENCOUNTER table for definitions.,56 PROCEDURES,PX_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"New to v2.0. -Date the procedure was performed.",57 -PROCEDURES,PX,RDBMS Text(11),SAS Char(11),,NO,,,,Procedure code.,58 +Date the procedure was performed.",57 +PROCEDURES,PX,RDBMS Text(11),SAS Char(11),,NO,,,,Procedure code.,58 PROCEDURES,PX_TYPE,RDBMS Text(2),SAS Char(2),,NO,,09;10;11;C2;C3;C4;H3;HC;LC;ND;RE;NI;UN;OT,"09 = ICD-9-CM 10 = ICD-10-PCS 11 = ICD-11-PCS @@ -255,7 +255,7 @@ Medications administered by clinicians can be captured in billing data and Elect We are now seeing NDCs captured as part of procedures because payers are demanding it for payment authorization. Inclusion of this code type enables those data partners that capture the NDC along with the procedure to include the data. -Please note: The ?Other? category is meant to identify internal use ontologies and codes.",59 +Please note: The ?Other? category is meant to identify internal use ontologies and codes.",59 PROCEDURES,PX_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,OD;BI;CL;NI;UN;OT,"OD=Order BI=Billing CL=Claim @@ -265,16 +265,16 @@ OT=Other","New to v2.0. Source of the procedure information. -Order and billing pertain to internal healthcare processes and data sources. Claim pertains to data from the bill fulfillment, generally data sources held by insurers and other health plans.",60 -PROCEDURES,RAW_PX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",61 -PROCEDURES,RAW_PX_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",62 +Order and billing pertain to internal healthcare processes and data sources. Claim pertains to data from the bill fulfillment, generally data sources held by insurers and other health plans.",60 +PROCEDURES,RAW_PX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",61 +PROCEDURES,RAW_PX_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",62 VITAL,VITALID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique VITAL record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. -(New field added to v3.0.)",63 -VITAL,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,64 -VITAL,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. This is an optional relationship; the ENCOUNTERID should generally be present if the vitals were measured as part of healthcare delivery captured by this datamart (guidance added in v3.0).,65 -VITAL,MEASURE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date of vitals measure.,66 -VITAL,MEASURE_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Time of vitals measure.,67 +(New field added to v3.0.)",63 +VITAL,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,64 +VITAL,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. This is an optional relationship; the ENCOUNTERID should generally be present if the vitals were measured as part of healthcare delivery captured by this datamart (guidance added in v3.0).,65 +VITAL,MEASURE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date of vitals measure.,66 +VITAL,MEASURE_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Time of vitals measure.,67 VITAL,VITAL_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,PR;PD;HC;HD;NI;UN;OT,"PR = Patient-reported PD=Patient device direct feed HC = Healthcare delivery setting @@ -285,20 +285,20 @@ OT = Other","Please note: The ?Patient-reported? category can include reporting v2.0 amendment: The new categorical value of PD and HD have been added. -v2.0 guidance added with slight modification in v3.0: If unknown whether data are received directly from a device feed, use the more general context (such as patient-reported or healthcare delivery setting).",68 -VITAL,HT,RDBMS Number(8),NUMERIC(8),,NO,INCH,,,"Height (in inches) measured by standing. Only populated if measure was taken on this date. If missing, this value should be null. Decimal precision is permissible. (Modification of wording made from ?blank? to ?null? in v3.0).",69 -VITAL,WT,RDBMS Number(8),NUMERIC(8),,NO,POUND,,,"Weight (in pounds). Only populated if measure was taken on this date. If missing, this value should be null. Decimal precision is permissible. (Modification of wording made from ?blank? to ?null? in v3.0).",70 -VITAL,DIASTOLIC,RDBMS Number(4),NUMERIC(),,NO,MMHG,,,"Diastolic blood pressure (in mmHg). Only populated if measure was taken on this date. If missing, this value should be null. (Modification of wording made from ?blank? to ?null? in v3.0).",71 -VITAL,SYSTOLIC,RDBMS Number(4),NUMERIC(4),,NO,MMHG,,,"Systolic blood pressure (in mmHg). Only populated if measure was taken on this date. If missing, this value should be null. (Modification of wording made from ?blank? to ?null? in v3.0).",72 +v2.0 guidance added with slight modification in v3.0: If unknown whether data are received directly from a device feed, use the more general context (such as patient-reported or healthcare delivery setting).",68 +VITAL,HT,RDBMS Number(8),NUMERIC(8),,NO,INCH,,,"Height (in inches) measured by standing. Only populated if measure was taken on this date. If missing, this value should be null. Decimal precision is permissible. (Modification of wording made from ?blank? to ?null? in v3.0).",69 +VITAL,WT,RDBMS Number(8),NUMERIC(8),,NO,POUND,,,"Weight (in pounds). Only populated if measure was taken on this date. If missing, this value should be null. Decimal precision is permissible. (Modification of wording made from ?blank? to ?null? in v3.0).",70 +VITAL,DIASTOLIC,RDBMS Number(4),NUMERIC(),,NO,MMHG,,,"Diastolic blood pressure (in mmHg). Only populated if measure was taken on this date. If missing, this value should be null. (Modification of wording made from ?blank? to ?null? in v3.0).",71 +VITAL,SYSTOLIC,RDBMS Number(4),NUMERIC(4),,NO,MMHG,,,"Systolic blood pressure (in mmHg). Only populated if measure was taken on this date. If missing, this value should be null. (Modification of wording made from ?blank? to ?null? in v3.0).",72 VITAL,ORIGINAL_BMI,RDBMS Number(8),NUMERIC(8),,NO,,,,"BMI if calculated in the source system. -Important: Please do not calculate BMI during CDM implementation. This field should only reflect originating source system calculations, if height and weight are not stored in the source.",73 +Important: Please do not calculate BMI during CDM implementation. This field should only reflect originating source system calculations, if height and weight are not stored in the source.",73 VITAL,BP_POSITION,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;NI;UN;OT,"01 = Sitting 02 = Standing 03 = Supine NI = No information UN = Unknown -OT = Other",Position for orthostatic blood pressure. This value should be null if blood pressure was not measured. (Modification of wording made from ?blank? to ?null? in v3.0).,74 +OT = Other",Position for orthostatic blood pressure. This value should be null if blood pressure was not measured. (Modification of wording made from ?blank? to ?null? in v3.0).,74 VITAL,SMOKING,RDBMS Text(2),SAS Char(2),,,,01;02;03;04;05;06;07;08;NI;UN;OT,"01=Current every day smoker 02=Current some day smoker 03=Former smoker @@ -321,7 +321,7 @@ Per Meaningful Use guidance, ??smoking status includes any form of tobacco that http://www.healthit.gov/sites/default/files/standards-certification/2014-edition-draft-test-procedures/170-314-a-11-smoking-status-2014-test-procedure-draft-v1.0.pdf [retrieved May 11, 2015] -",75 +",75 VITAL,TOBACCO,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;06;NI;UN;OT,"01=Current user 02=Never 03=Quit/former user @@ -332,7 +332,7 @@ UN=Unknown OT=Other ","This field is new to v2.0 with revised value set and field definition in v3.0. -Indicator for any form of tobacco.",76 +Indicator for any form of tobacco.",76 VITAL,TOBACCO_TYPE,RDBMS Text(2),SAS Char(2),,NO,,"01;02;03;04;05;NI;UN;OT ","01=Smoked tobacco only 02=Non-smoked tobacco only @@ -344,35 +344,35 @@ UN=Unknown OT=Other ","This field is new to v2.0, with revised value set in v3.0. -Type(s) of tobacco used.",77 -VITAL,RAW_DIASTOLIC,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to formatting into the PCORnet CDM.",78 -VITAL,RAW_SYSTOLIC,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to formatting into the PCORnet CDM.",79 -VITAL,RAW_BP_POSITION,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",80 -VITAL,RAW_SMOKING,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set. New to v3.0.",81 -VITAL,RAW_TOBACCO,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set. New to v2.0.",82 -VITAL,RAW_TOBACCO_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set. New to v2.0.",83 -DISPENSING,DISPENSINGID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",84 -DISPENSING,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,85 -DISPENSING,PRESCRIBINGID,RDBMS Text(x),SAS Char(x),,,,,,"This is an optional relationship to the PRESCRIBING table, and may not be generally available. One prescribing order may generate multiple dispensing records. New field added in v3.0.",86 -DISPENSING,DISPENSE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Dispensing date (as close as possible to date the person received the dispensing).,87 +Type(s) of tobacco used.",77 +VITAL,RAW_DIASTOLIC,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to formatting into the PCORnet CDM.",78 +VITAL,RAW_SYSTOLIC,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to formatting into the PCORnet CDM.",79 +VITAL,RAW_BP_POSITION,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",80 +VITAL,RAW_SMOKING,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set. New to v3.0.",81 +VITAL,RAW_TOBACCO,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set. New to v2.0.",82 +VITAL,RAW_TOBACCO_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set. New to v2.0.",83 +DISPENSING,DISPENSINGID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",84 +DISPENSING,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,85 +DISPENSING,PRESCRIBINGID,RDBMS Text(x),SAS Char(x),,,,,,"This is an optional relationship to the PRESCRIBING table, and may not be generally available. One prescribing order may generate multiple dispensing records. New field added in v3.0.",86 +DISPENSING,DISPENSE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Dispensing date (as close as possible to date the person received the dispensing).,87 DISPENSING,NDC,RDBMS Text(11),SAS Char(11),,NO,,,,"National Drug Code in the 11-digit, no-dash, HIPAA format. Please expunge any place holders (such as dashes or extra digits). -If needed, guidance on normalization for other forms of NDC can be found: http://www.nlm.nih.gov/research/umls/rxnorm/docs/2012/rxnorm_doco_full_2012-1.html (see section 6)",88 +If needed, guidance on normalization for other forms of NDC can be found: http://www.nlm.nih.gov/research/umls/rxnorm/docs/2012/rxnorm_doco_full_2012-1.html (see section 6)",88 DISPENSING,DISPENSE_SUP,RDBMS Number(8),Numeric(8),,NO,DAYS,,,"Days supply. Number of days that the medication supports based on the number of doses as reported by the pharmacist. This amount is typically found on the dispensing record. Integer values are expected. -Important: Please do not calculate during CDM implementation. This field should only reflect originating source system calculations.",89 +Important: Please do not calculate during CDM implementation. This field should only reflect originating source system calculations.",89 DISPENSING,DISPENSE_AMT,RDBMS Number(8),Numeric(8),,NO,UNITS,,,"Number of units (pills, tablets, vials) dispensed. Net amount per NDC per dispensing. This amount is typically found on the dispensing record. Positive values are expected. Important: Please do not calculate during CDM implementation. This field should only reflect originating source system calculations. -",90 -DISPENSING,RAW_NDC,RDBMS Text(x),Numeric(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",91 +",90 +DISPENSING,RAW_NDC,RDBMS Text(x),Numeric(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",91 LAB_RESULT_CM ,LAB_RESULT_CM_ID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique LAB_RESULT_CM record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. -(New field added to v3.0.)",92 -LAB_RESULT_CM ,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,93 -LAB_RESULT_CM ,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier used to link across tables. This is an optional field, and should only be populated if the lab was collected as part of a healthcare encounter.",94 +(New field added to v3.0.)",92 +LAB_RESULT_CM ,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,93 +LAB_RESULT_CM ,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier used to link across tables. This is an optional field, and should only be populated if the lab was collected as part of a healthcare encounter.",94 LAB_RESULT_CM ,LAB_NAME,RDBMS Text(10),SAS Char(10),,NO,,A1C;CK;CK_MB;CK_MBI;CREATININE;HGB;LDL;INR;TROP_I;TROP_T_QL;TROP_T_QN,"A1C=Hemoglobin A1c CK=Creatine kinase total CK_MB=Creatine kinase MB @@ -385,7 +385,7 @@ TROP_I=Troponin I cardiac TROP_T_QL=Troponin T cardiac (qualitative) TROP_T_QN=Troponin T cardiac (quantitative)","Laboratory result common measure, a categorical identification for the type of test, which is harmonized across all contributing data partners. -Please note that it is possible for more than one LOINC code, CPT code, and/or local code to be associated with one LAB_NAME.",95 +Please note that it is possible for more than one LOINC code, CPT code, and/or local code to be associated with one LAB_NAME.",95 LAB_RESULT_CM ,SPECIMEN_SOURCE,RDBMS Text(10),SAS Char(10),,NO,,BLOOD;CSF;PLASMA;PPP;SERUM;SR_PLS;URINE;NI;UN;OT,"BLOOD=blood CSF=cerebrospinal fluid PLASMA=plasma @@ -395,23 +395,23 @@ SR_PLS=serum/plasma URINE=urine NI=No information UN=Unknown -OT=Other",Specimen source. All records will have a specimen source; some tests have several possible values for SPECIMEN_SOURCE. Please see the reference tables for additional details.,96 +OT=Other",Specimen source. All records will have a specimen source; some tests have several possible values for SPECIMEN_SOURCE. Please see the reference tables for additional details.,96 LAB_RESULT_CM ,LAB_LOINC,RDBMS Text(10),SAS Char(10),,NO,,,,"Logical Observation Identifiers, Names, and Codes (LOINC) from the Regenstrief Institute. Results with local versions of LOINC codes (e.g., LOINC candidate codes) should be included in the RAW_table field, but the LOINC variable should be set to missing. Current LOINC codes are from 3-7 characters long but Regenstrief suggests a length of 10 for future growth. The last digit of the LOINC code is a check digit and is always preceded by a hyphen. All parts of the LOINC code, including the hyphen, must be included. Do not pad the LOINC code with leading zeros. Please see the LOINC reference table for known LOINC codes for each LAB_NAME. -",97 +",97 LAB_RESULT_CM ,PRIORITY,RDBMS Text(2),SAS Char(2),,NO,,E;R;S;NI;UN;OT,"E=Expedite R=Routine S=Stat NI=No information UN=Unknown -OT=Other",Immediacy of test. The intent of this variable is to determine whether the test was obtained as part of routine care or as an emergent/urgent diagnostic test (designated as Stat or Expedite).,98 +OT=Other",Immediacy of test. The intent of this variable is to determine whether the test was obtained as part of routine care or as an emergent/urgent diagnostic test (designated as Stat or Expedite).,98 LAB_RESULT_CM ,RESULT_LOC,RDBMS Text(2),SAS Char(2),,NO,,L;P;NI;UN;OT,"L=Lab P=Point of Care NI=No information UN=Unknown -OT=Other","Location of the test result. Point of Care locations may include anticoagulation clinic, newborn nursery, finger stick in provider office, or home. The default value is ?L? unless the result is Point of Care. There should not be any missing values.",99 -LAB_RESULT_CM ,LAB_PX,RDBMS Text(11),SAS Char(11),,NO,,,,"Optional variable for local and standard procedure codes, used to identify the originating order for the lab test.",100 +OT=Other","Location of the test result. Point of Care locations may include anticoagulation clinic, newborn nursery, finger stick in provider office, or home. The default value is ?L? unless the result is Point of Care. There should not be any missing values.",99 +LAB_RESULT_CM ,LAB_PX,RDBMS Text(11),SAS Char(11),,NO,,,,"Optional variable for local and standard procedure codes, used to identify the originating order for the lab test.",100 LAB_RESULT_CM ,LAB_PX_TYPE,RDBMS Text(2),SAS Char(2),,NO,,09;10;11;C2;C3;C4;H3;HC;LC;ND;RE;NI;UN;OT,"09=ICD-9-CM 10=ICD-10-PCS 11=ICD-11-PCS @@ -425,12 +425,12 @@ ND=NDC RE=Revenue NI=No information UN=Unknown -OT=Other","Procedure code type, if applicable.",101 -LAB_RESULT_CM ,LAB_ORDER_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date test was ordered.,102 -LAB_RESULT_CM ,SPECIMEN_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date specimen was collected.,103 -LAB_RESULT_CM ,SPECIMEN_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Time specimen was collected.,104 -LAB_RESULT_CM ,RESULT_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Result date.,105 -LAB_RESULT_CM ,RESULT_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Result time.,106 +OT=Other","Procedure code type, if applicable.",101 +LAB_RESULT_CM ,LAB_ORDER_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date test was ordered.,102 +LAB_RESULT_CM ,SPECIMEN_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date specimen was collected.,103 +LAB_RESULT_CM ,SPECIMEN_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Time specimen was collected.,104 +LAB_RESULT_CM ,RESULT_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Result date.,105 +LAB_RESULT_CM ,RESULT_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Result time.,106 LAB_RESULT_CM ,RESULT_QUAL,RDBMS Text(12),SAS Char(12),,NO,,"BORDERLINE;POSITIVE;NEGATIVE;UNDETERMINED;NI=No information;UN=Unknown;OT=Other ","BORDERLINE POSITIVE @@ -438,8 +438,8 @@ NEGATIVE UNDETERMINED NI=No information UN=Unknown -OT=Other",Standardized result for qualitative results. This variable should be NI for quantitative results. Please see the reference tables for additional details and information on acceptable values for each qualitative LAB_NAME.,107 -LAB_RESULT_CM ,RESULT_NUM,RDBMS Number(8),SAS Char(8),,NO,,,,Standardized/converted result for quantitative results. This variable should be null for qualitative results. Please see the reference tables for additional details. (Modification of wording made from ?blank? to ?null? in v3.0).,108 +OT=Other",Standardized result for qualitative results. This variable should be NI for quantitative results. Please see the reference tables for additional details and information on acceptable values for each qualitative LAB_NAME.,107 +LAB_RESULT_CM ,RESULT_NUM,RDBMS Number(8),SAS Char(8),,NO,,,,Standardized/converted result for quantitative results. This variable should be null for qualitative results. Please see the reference tables for additional details. (Modification of wording made from ?blank? to ?null? in v3.0).,108 LAB_RESULT_CM ,RESULT_MODIFIER,RDBMS Text(2),SAS Char(2),,NO,,EQ;GE;GT;LE;LT;TX;NI;UN;OT,"EQ=Equal GE=Greater than or equal to GT=Greater than @@ -450,9 +450,9 @@ NI=No information UN=Unknown OT=Other","Modifier for result values. Any symbols in the RAW_RESULT value should be reflected in the RESULT_MODIFIER variable. -For example, if the original source data value is ""<=200"" then RAW_RESULT=200 and RESULT_MODIFIER=LE. If the original source data value is text then RESULT_MODIFIER=TX. If the original source data value is a numeric value then RESULT_MODIFIER=EQ.",109 -LAB_RESULT_CM ,RESULT_UNIT,RDBMS Text(11),SAS Char(11),,NO,,,,Converted/standardized units for the result. Please see the standard abbreviations reference table for additional details.,110 -LAB_RESULT_CM ,NORM_RANGE_LOW,RDBMS Text(10),SAS Char(10),,NO,,,,"Lower bound of the normal range assigned by the laboratory. Value should only contain the value of the lower bound. The symbols >, <, >=, <= should be removed. For example, if the normal range for a test is >100 and <300, then ""100"" should be entered.",111 +For example, if the original source data value is ""<=200"" then RAW_RESULT=200 and RESULT_MODIFIER=LE. If the original source data value is text then RESULT_MODIFIER=TX. If the original source data value is a numeric value then RESULT_MODIFIER=EQ.",109 +LAB_RESULT_CM ,RESULT_UNIT,RDBMS Text(11),SAS Char(11),,NO,,,,Converted/standardized units for the result. Please see the standard abbreviations reference table for additional details.,110 +LAB_RESULT_CM ,NORM_RANGE_LOW,RDBMS Text(10),SAS Char(10),,NO,,,,"Lower bound of the normal range assigned by the laboratory. Value should only contain the value of the lower bound. The symbols >, <, >=, <= should be removed. For example, if the normal range for a test is >100 and <300, then ""100"" should be entered.",111 LAB_RESULT_CM ,NORM_MODIFIER_LOW,RDBMS Text(2),SAS Char(2),,NO,,EQ;GE;GT;NO;NI;UN;OT,"EQ=Equal GE=Greater than or equal to GT=Greater than @@ -471,8 +471,8 @@ For numeric results one of the following needs to be true: For numeric results one of the following needs to be true: 1) Both MODIFIER_LOW and MODIFIER_HIGH contain EQ (e.g. normal values fall in the range 3-10) 2) MODIFIER_LOW contains GT or GE and MODIFIER_HIGH contains NO (e.g. normal values are >3 with no upper boundary) -3) MODIFIER_HIGH contains LT or LE and MODIFIER_LOW contains NO (e.g. normal values are <=10 with no lower boundary)",112 -LAB_RESULT_CM ,NORM_RANGE_HIGH,RDBMS Text(10),SAS Char(10),,NO,,,,"Upper bound of the normal range assigned by the laboratory. Value should only contain the value of the upper bound. The symbols >, <, >=, <= should be removed. For example, if the normal range for a test is >100 and <300, then ""300"" should be entered.",113 +3) MODIFIER_HIGH contains LT or LE and MODIFIER_LOW contains NO (e.g. normal values are <=10 with no lower boundary)",112 +LAB_RESULT_CM ,NORM_RANGE_HIGH,RDBMS Text(10),SAS Char(10),,NO,,,,"Upper bound of the normal range assigned by the laboratory. Value should only contain the value of the upper bound. The symbols >, <, >=, <= should be removed. For example, if the normal range for a test is >100 and <300, then ""300"" should be entered.",113 LAB_RESULT_CM ,NORM_MODIFIER_HIGH,RDBMS Text(2),SAS Char(2),,NO,,EQ;LE;LT;NO;NI;UN;OT,"EQ=Equal LE=Less than or equal to LT=Less than @@ -486,7 +486,7 @@ v3.0 modification made to field name. For numeric results one of the following needs to be true: 1) Both MODIFIER_LOW and MODIFIER_HIGH contain EQ (e.g. normal values fall in the range 3-10) 2) MODIFIER_LOW contains GT or GE and MODIFIER_HIGH contains NO (e.g. normal values are >3 with no upper boundary) -3) MODIFIER_HIGH contains LT or LE and MODIFIER_LOW contains NO (e.g. normal values are <=10 with no lower boundary)",114 +3) MODIFIER_HIGH contains LT or LE and MODIFIER_LOW contains NO (e.g. normal values are <=10 with no lower boundary)",114 LAB_RESULT_CM ,ABN_IND,RDBMS Text(2),SAS Char(2),,NO,,AB;AH;AL;CH;CL;CR;IN;NL;NI;UN;OT,"AB=Abnormal AH=Abnormally high AL=Abnormally low @@ -497,24 +497,24 @@ IN=Inconclusive NL=Normal NI=No information UN=Unknown -OT=Other",Abnormal result indicator. This value comes from the source data; do not apply logic to create it.,115 -LAB_RESULT_CM ,RAW_LAB_NAME,RDBMS Text(x),SAS Char(x),,NO,,,,"Local code related to an individual lab test. This variable will not be used in queries, but may be used by local programmers to associate a record with a particular LAB_NAME.",116 -LAB_RESULT_CM ,RAW_LAB_CODE,RDBMS Text(x),SAS Char(x),,NO,,,,"Local code related to an individual lab test. This variable will not be used in queries, but may be used by local programmers to associate a record with a particular LAB_NAME.",117 -LAB_RESULT_CM ,RAW_PANEL,RDBMS Text(x),SAS Char(x),,NO,,,,"Local code related to a battery or panel of lab tests. This variable will not be used in queries, but may be used by local programmers to associate a record with a particular LAB_NAME.",118 -LAB_RESULT_CM ,RAW_RESULT,RDBMS Text(x),SAS Char(x),,NO,,,,"The original test result value as seen in your source data. Values may include a decimal point, a sign or text (e.g., POSITIVE, NEGATIVE, DETECTED). The symbols >, <, >=, <= should be removed from the value and stored in the Modifier variable instead.",119 -LAB_RESULT_CM ,RAW_UNIT,RDBMS Text(x),SAS Char(x),,NO,,,,Original units for the result in your source data.,120 -LAB_RESULT_CM ,RAW_ORDER_DEPT,RDBMS Text(x),SAS Char(x),,NO,,,,Local code for ordering provider department.,121 -LAB_RESULT_CM ,RAW_FACILITY_CODE,RDBMS Text(x),SAS Char(x),,NO,,,,Local facility code that identifies the hospital or clinic. Taken from facility claims.,122 -CONDITION,CONDITIONID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",123 -CONDITION,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,124 +OT=Other",Abnormal result indicator. This value comes from the source data; do not apply logic to create it.,115 +LAB_RESULT_CM ,RAW_LAB_NAME,RDBMS Text(x),SAS Char(x),,NO,,,,"Local code related to an individual lab test. This variable will not be used in queries, but may be used by local programmers to associate a record with a particular LAB_NAME.",116 +LAB_RESULT_CM ,RAW_LAB_CODE,RDBMS Text(x),SAS Char(x),,NO,,,,"Local code related to an individual lab test. This variable will not be used in queries, but may be used by local programmers to associate a record with a particular LAB_NAME.",117 +LAB_RESULT_CM ,RAW_PANEL,RDBMS Text(x),SAS Char(x),,NO,,,,"Local code related to a battery or panel of lab tests. This variable will not be used in queries, but may be used by local programmers to associate a record with a particular LAB_NAME.",118 +LAB_RESULT_CM ,RAW_RESULT,RDBMS Text(x),SAS Char(x),,NO,,,,"The original test result value as seen in your source data. Values may include a decimal point, a sign or text (e.g., POSITIVE, NEGATIVE, DETECTED). The symbols >, <, >=, <= should be removed from the value and stored in the Modifier variable instead.",119 +LAB_RESULT_CM ,RAW_UNIT,RDBMS Text(x),SAS Char(x),,NO,,,,Original units for the result in your source data.,120 +LAB_RESULT_CM ,RAW_ORDER_DEPT,RDBMS Text(x),SAS Char(x),,NO,,,,Local code for ordering provider department.,121 +LAB_RESULT_CM ,RAW_FACILITY_CODE,RDBMS Text(x),SAS Char(x),,NO,,,,Local facility code that identifies the hospital or clinic. Taken from facility claims.,122 +CONDITION,CONDITIONID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",123 +CONDITION,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,124 CONDITION,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier used to link across tables. This is an optional field, and should only be populated if the item was collected as part of a healthcare encounter. -If more than one encounter association is present, this field should be populated with the ID of the encounter when the condition was first entered into the system. However, please note that many conditions may be recorded outside of an encounter context. (Guidance added in v3.0.)",125 -CONDITION,REPORT_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date condition was noted, which may be the date when it was recorded by a provider or nurse, or the date on which the patient reported it. Please note that this date may not correspond to onset date. (Additional guidance added in v3.0.)",126 -CONDITION,RESOLVE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date condition was resolved, if resolution of a transient condition has been achieved. A resolution date is not generally expected for chronic conditions, even if the condition is managed.",127 +If more than one encounter association is present, this field should be populated with the ID of the encounter when the condition was first entered into the system. However, please note that many conditions may be recorded outside of an encounter context. (Guidance added in v3.0.)",125 +CONDITION,REPORT_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date condition was noted, which may be the date when it was recorded by a provider or nurse, or the date on which the patient reported it. Please note that this date may not correspond to onset date. (Additional guidance added in v3.0.)",126 +CONDITION,RESOLVE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date condition was resolved, if resolution of a transient condition has been achieved. A resolution date is not generally expected for chronic conditions, even if the condition is managed.",127 CONDITION,ONSET_DATE,RDBMS Date,SAS Date (Numeric),,,,,,"New field added in v3.0. -Please note that onset date is a very precise concept. Please do not map data unless they precisely match this definition. The REPORT_DATE concept may be a better fit for many systems.",128 +Please note that onset date is a very precise concept. Please do not map data unless they precisely match this definition. The REPORT_DATE concept may be a better fit for many systems.",128 CONDITION,CONDITION_STATUS,RDBMS Text(2),SAS Char(2),,NO,,AC;RS;IN;NI;UN;OT,"AC=Active RS=Resolved IN=Inactive @@ -522,10 +522,10 @@ NI=No information UN=Unknown OT=Other","Condition status corresponding with REPORT_DATE. -Guidance: The value of IN=Inactive may be used in situations where a condition is not resolved, but is not currently active (for example, psoriasis).",129 +Guidance: The value of IN=Inactive may be used in situations where a condition is not resolved, but is not currently active (for example, psoriasis).",129 CONDITION,CONDITION,RDBMS Text(18),SAS Char(18),,NO,,,,"Condition code. -Leading zeroes and different levels of decimal precision are permissible in this field. Please populate the exact textual value of this diagnosis code, but remove source-specific suffixes and prefixes. Other codes should be listed as recorded in the source data.",130 +Leading zeroes and different levels of decimal precision are permissible in this field. Please populate the exact textual value of this diagnosis code, but remove source-specific suffixes and prefixes. Other codes should be listed as recorded in the source data.",130 CONDITION,CONDITION_TYPE,RDBMS Text(2),SAS Char(2),,NO,,09;10;11;SM;HP;AG;NI;UN;OT,"09=ICD-9-CM 10=ICD-10-CM 11=ICD-11-CM @@ -538,7 +538,7 @@ OT=Other","Condition code type. Please note: The ?Other? category is meant to identify internal use ontologies and codes. -v3.0 amendment: The new categorical value of AG has been added. ",131 +v3.0 amendment: The new categorical value of AG has been added. ",131 CONDITION,CONDITION_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,PR;HC;RG;PC;NI;UN;OT,"PR=Patient-reported medical history HC=Healthcare problem list RG=Registry cohort @@ -551,14 +551,14 @@ Guidance: ?Registry cohort? generally refers to cohorts of patients flagged with ?Patient-reported? can include self-reported medical history and/or current medical conditions, not captured via healthcare problem lists or registry cohorts. -v3.0 amendment: The new categorical value of PC has been added.",132 -CONDITION,RAW_CONDITION_STATUS,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",133 -CONDITION,RAW_CONDITION,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",134 -CONDITION,RAW_CONDITION_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",135 -CONDITION,RAW_CONDITION_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",136 -PRO_CM,PRO_CM_ID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",137 -PRO_CM,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier for the patient for whom the PRO response was captured. Used to link across tables.,138 -PRO_CM,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier used to link across tables. This is an optional field, and should only be populated if the item was collected as part of a healthcare encounter.",139 +v3.0 amendment: The new categorical value of PC has been added.",132 +CONDITION,RAW_CONDITION_STATUS,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",133 +CONDITION,RAW_CONDITION,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",134 +CONDITION,RAW_CONDITION_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",135 +CONDITION,RAW_CONDITION_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",136 +PRO_CM,PRO_CM_ID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",137 +PRO_CM,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier for the patient for whom the PRO response was captured. Used to link across tables.,138 +PRO_CM,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier used to link across tables. This is an optional field, and should only be populated if the item was collected as part of a healthcare encounter.",139 PRO_CM,PRO_ITEM,RDBMS Text(7),SAS Char(7),,NO,,PN_0001;PN_0002;PN_0003;PN_0004;PN_0005;PN_0006;PN_0007;PN_0008;PN_0009;PN_0010;PN_0011;PN_0012;PN_0013;PN_0014;PN_0015;PN_0016;PN_0017;PN_0018;PN_0019;PN_0020;PN_0021,"PN_0001=GLOBAL01 PN_0002=GLOBAL02 PN_0003=GLOBAL06 @@ -579,21 +579,21 @@ PN_0017=GLOBAL04 PN_0018=EDANX53 PN_0019=SAMHSA PN_0020=CAHPS 4.0 -PN_0021=PA070",PCORnet identifier for the specific Common Measure item. Please see the Common Measures reference table for more details.,140 +PN_0021=PA070",PCORnet identifier for the specific Common Measure item. Please see the Common Measures reference table for more details.,140 PRO_CM,PRO_LOINC,RDBMS Text(10),SAS Char(10),,NO,,,,"LOINC code for item context and stem. Please see the reference table for known LOINC codes for each common measure. Logical Observation Identifiers, Names, and Codes (LOINC) from the Regenstrief Institute. Results with local versions of LOINC codes (e.g., LOINC candidate codes) should be included in the RAW_table field, but the PRO_LOINC variable should be set to missing. Current LOINC codes are from 3-7 characters long but Regenstrief suggests a length of 10 for future growth. The last digit of the LOINC code is a check digit and is always preceded by a hyphen. All parts of the LOINC code, including the hyphen, must be included. Do not pad the LOINC code with leading zeros. -",141 -PRO_CM,PRO_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,The date of the response.,142 -PRO_CM,PRO_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,The time of the response.,143 -PRO_CM,PRO_RESPONSE,NUMBER(8),Numeric(8),,NO,,,,The numeric response recorded for the item. Please see the Common Measures reference table for the list of valid responses for each item.,144 +",141 +PRO_CM,PRO_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,The date of the response.,142 +PRO_CM,PRO_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,The time of the response.,143 +PRO_CM,PRO_RESPONSE,NUMBER(8),Numeric(8),,NO,,,,The numeric response recorded for the item. Please see the Common Measures reference table for the list of valid responses for each item.,144 PRO_CM,PRO_METHOD,RDBMS Text(2),SAS Char(2),,NO,,PA;EC;PH;IV;NI;UN;OT,"PA=Paper EC=Electronic PH=Telephonic IV=Telephonic with interactive voice response (IVR) technology NI=No information UN=Unknown -OT=Other","Method of administration. Electronic includes responses captured via a personal or tablet computer, at web kiosks, or via a smartphone.",145 +OT=Other","Method of administration. Electronic includes responses captured via a personal or tablet computer, at web kiosks, or via a smartphone.",145 PRO_CM,PRO_MODE,RDBMS Text(2),SAS Char(2),,NO,,"SF;SA;PR;PA;NI;UN;OT ","SF=Self without assistance SA= Self with assistance @@ -601,26 +601,26 @@ PR=Proxy without assistance PA=Proxy with assistance NI=No information UN=Unknown -OT=Other","The person who responded on behalf of the patient for whom the response was captured. A proxy report is a measurement based on a report by someone other than the patient reporting as if he or she is the patient, such as a parent responding for a child, or a caregiver responding for an individual unable to report for themselves. Assistance excludes providing interpretation of the patient?s response. ",146 +OT=Other","The person who responded on behalf of the patient for whom the response was captured. A proxy report is a measurement based on a report by someone other than the patient reporting as if he or she is the patient, such as a parent responding for a child, or a caregiver responding for an individual unable to report for themselves. Assistance excludes providing interpretation of the patient?s response. ",146 PRO_CM,PRO_CAT,RDBMS Text(2),SAS Char(2),,NO,,"Y;N;NI;UN;OT ","Y=Yes N=No NI=No information UN=Unknown -OT=Other",Indicates whether Computer Adaptive Testing (CAT) was used to administer the survey or instrument that the item was part of. May apply to electronic (EC) and telephonic (PH or IV) modes. ,147 -PRO_CM,RAW_PRO_CODE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating code, such as LOINC candidate codes that have not yet been adopted",148 -PRO_CM,RAW_PRO_RESPONSE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",149 -PRESCRIBING,PRESCRIBINGID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary identifier for each unique PRESCRIBING record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID.",150 -PRESCRIBING,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,151 -PRESCRIBING,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. This is an optional relationship; the ENCOUNTERID should be present if the prescribing activity is directly associated with an encounter.,152 -PRESCRIBING,RX_PROVIDERID,RDBMS Text(x),SAS Char(x),,NO,,,,Provider code for the provider who prescribed the medication. The provider code is a pseudoidentifier with a consistent crosswalk to the real identifier. ,153 -PRESCRIBING,RX_ORDER_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Order date of the prescription by the provider.,154 -PRESCRIBING,RX_ORDER_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,,,,Order time of the prescription by the provider.,155 -PRESCRIBING,RX_START_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Start date of order. This attribute may not be consistent with the date on which the patient actually begin taking the medication.,156 -PRESCRIBING,RX_END_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,End date of order (if available).,157 -PRESCRIBING,RX_QUANTITY,RDBMS Number(8),Numeric(8),,NO,,,,Quantity ordered.,158 -PRESCRIBING,RX_REFILLS,RDBMS Number(8),Numeric(8),,NO,,,,"Number of refills ordered (not including the original prescription). If no refills are ordered, the value should be zero.",159 -PRESCRIBING,RX_DAYS_SUPPLY,RDBMS Number(8),Numeric(8),,NO,,,,"Number of days supply ordered, as specified by the prescription.",160 +OT=Other",Indicates whether Computer Adaptive Testing (CAT) was used to administer the survey or instrument that the item was part of. May apply to electronic (EC) and telephonic (PH or IV) modes. ,147 +PRO_CM,RAW_PRO_CODE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating code, such as LOINC candidate codes that have not yet been adopted",148 +PRO_CM,RAW_PRO_RESPONSE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",149 +PRESCRIBING,PRESCRIBINGID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary identifier for each unique PRESCRIBING record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID.",150 +PRESCRIBING,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,151 +PRESCRIBING,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. This is an optional relationship; the ENCOUNTERID should be present if the prescribing activity is directly associated with an encounter.,152 +PRESCRIBING,RX_PROVIDERID,RDBMS Text(x),SAS Char(x),,NO,,,,Provider code for the provider who prescribed the medication. The provider code is a pseudoidentifier with a consistent crosswalk to the real identifier. ,153 +PRESCRIBING,RX_ORDER_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Order date of the prescription by the provider.,154 +PRESCRIBING,RX_ORDER_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,,,,Order time of the prescription by the provider.,155 +PRESCRIBING,RX_START_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Start date of order. This attribute may not be consistent with the date on which the patient actually begin taking the medication.,156 +PRESCRIBING,RX_END_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,End date of order (if available).,157 +PRESCRIBING,RX_QUANTITY,RDBMS Number(8),Numeric(8),,NO,,,,Quantity ordered.,158 +PRESCRIBING,RX_REFILLS,RDBMS Number(8),Numeric(8),,NO,,,,"Number of refills ordered (not including the original prescription). If no refills are ordered, the value should be zero.",159 +PRESCRIBING,RX_DAYS_SUPPLY,RDBMS Number(8),Numeric(8),,NO,,,,"Number of days supply ordered, as specified by the prescription.",160 PRESCRIBING,RX_FREQUENCY,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;05;06;07;08;09;NI;UN;OT,"01=Every day 02=Two times a day (BID) 03=Three times a day (TID) @@ -633,13 +633,13 @@ PRESCRIBING,RX_FREQUENCY,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;05;06;07;08; NI=No information UN=Unknown OT=Other -",Specified frequency of medication.,161 +",Specified frequency of medication.,161 PRESCRIBING,RX_BASIS,RDBMS Text(2),SAS Char(2),,NO,,01;02;NI;UN;OT,"01=Dispensing 02=Administration NI=No information UN=Unknown OT=Other -",Basis of the medication order,162 +",Basis of the medication order,162 PRESCRIBING,RXNORM_CUI,RDBMS Number (8),Numeric(8),,NO,,,,"Where an RxNorm mapping exists for the source medication, this field contains the RxNorm concept identifier (CUI) at the highest possible specificity. If more than one option exists for mapping, the following ordered strategy may be adopted: @@ -647,28 +647,28 @@ If more than one option exists for mapping, the following ordered strategy may b 2)Semantic Branded clinical drug 3)Generic drug pack 4)Branded drug pack -",163 -PRESCRIBING,RAW_RX_MED_NAME,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating, full textual medication name from the source.",164 -PRESCRIBING,RAW_RX_FREQUENCY,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",165 -PRESCRIBING,RAW_RXNORM_CUI,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",166 -PCORNET_TRIAL,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,167 -PCORNET_TRIAL,TRIALID,RDBMS Text(20),SAS Char(20),,NO,,,,Each TRIALID is assigned by the PCORnet trial?s coordinating center.,168 +",163 +PRESCRIBING,RAW_RX_MED_NAME,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating, full textual medication name from the source.",164 +PRESCRIBING,RAW_RX_FREQUENCY,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",165 +PRESCRIBING,RAW_RXNORM_CUI,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",166 +PCORNET_TRIAL,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,167 +PCORNET_TRIAL,TRIALID,RDBMS Text(20),SAS Char(20),,NO,,,,Each TRIALID is assigned by the PCORnet trial?s coordinating center.,168 PCORNET_TRIAL,PARTICIPANTID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary person-level identifier used to uniquely identify a participant in a PCORnet trial. PARTICIPANTID is never repeated or reused for a specific clinical trial, and is generally assigned by trial-specific processes. It may be the same as a randomization ID. -",169 -PCORNET_TRIAL,TRIAL_SITEID,RDBMS Text(x),SAS Char(x),,NO,,,,Each TRIAL_SITEID is assigned by the PCORnet trial coordinating center.,170 -PCORNET_TRIAL,TRIAL_ENROLL_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Date on which the participant enrolled in the trial (generally coincides with trial consent process).,171 -PCORNET_TRIAL,TRIAL_END_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Date on which the participant completes participation in the trial.,172 -PCORNET_TRIAL,TRIAL_WITHDRAW_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,"If applicable, date on which the participant withdraws consent from the trial.",173 +",169 +PCORNET_TRIAL,TRIAL_SITEID,RDBMS Text(x),SAS Char(x),,NO,,,,Each TRIAL_SITEID is assigned by the PCORnet trial coordinating center.,170 +PCORNET_TRIAL,TRIAL_ENROLL_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Date on which the participant enrolled in the trial (generally coincides with trial consent process).,171 +PCORNET_TRIAL,TRIAL_END_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Date on which the participant completes participation in the trial.,172 +PCORNET_TRIAL,TRIAL_WITHDRAW_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,"If applicable, date on which the participant withdraws consent from the trial.",173 PCORNET_TRIAL,TRIAL_INVITE_CODE,RDBMS Text(20),SAS Char(20),,NO,,,,"Textual strings used to uniquely identify invitations sent to potential participants, and allows acceptances to be associated back to the originating source. Where used, there should generally be a unique combination of PATID, TRIAL_NAME, and INVITE_CODE within each datamart. For example, this might include ?co-enrollment ID strings? for e-mail invites or ?verification codes? for letter invites. -",174 -DEATH,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,175 -DEATH,DEATH_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Date of death.,176 +",174 +DEATH,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,175 +DEATH,DEATH_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Date of death.,176 DEATH,DEATH_DATE_IMPUTE,RDBMS Text(2),SAS Char(2),,NO,,B;D;M;N;NI;UN;OT,"B=Both month and day imputed D=Day imputed M=Month imputed @@ -676,7 +676,7 @@ N=Not imputed NI=No information UN=Unknown OT=Other -","When date of death is imputed, this field indicates which parts of the date were imputed.",177 +","When date of death is imputed, this field indicates which parts of the date were imputed.",177 DEATH,DEATH_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,L;N;D;S;T;NI;UN;OT,"L=Other, locally defined N=National Death Index D=Social Security @@ -685,7 +685,7 @@ T=Tumor data NI=No information UN=Unknown OT=Other -","Guidance: ?Other, locally defined? may be used to indicate presence of deaths reported from EHR systems, such as in-patient hospital deaths or dead on arrival.",178 +","Guidance: ?Other, locally defined? may be used to indicate presence of deaths reported from EHR systems, such as in-patient hospital deaths or dead on arrival.",178 DEATH,DEATH_MATCH_CONFIDENCE,RDBMS Text(2),SAS Char(2),,NO,,E;F;P;NI;UN;OT,"E=Excellent F=Fair P=Poor @@ -695,15 +695,15 @@ OT=Other ","For situations where a probabilistic patient matching strategy is used, this field indicates the confidence that the patient drawn from external source data represents the actual patient. Should not be present where DEATH_SOURCE is L (locally-defined). May not be applicable for DEATH_SOURCE=T (tumor registry data) -",179 -DEATH_CAUSE,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,180 -DEATH_CAUSE,DEATH_CAUSE,RDBMS Text(8),SAS Char(8),,NO,,,,Cause of death code. Please include the decimal point in ICD codes (if any).,181 +",179 +DEATH_CAUSE,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,180 +DEATH_CAUSE,DEATH_CAUSE,RDBMS Text(8),SAS Char(8),,NO,,,,Cause of death code. Please include the decimal point in ICD codes (if any).,181 DEATH_CAUSE,DEATH_CAUSE_CODE,RDBMS Text(2),SAS Char(2),,NO,,09;10;NI;UN;OT,"09=ICD-9 10=ICD-10 NI=No information UN=Unknown OT=Other -",Cause of death code type.,182 +",Cause of death code type.,182 DEATH_CAUSE,DEATH_CAUSE_TYPE,RDBMS Text(2),SAS Char(2),,NO,,C;I;O;U;NI;UN;OT,"C=Contributory I=Immediate/Primary O=Other @@ -711,7 +711,7 @@ U=Underlying NI=No information UN=Unknown OT=Other -",Cause of death type. There should be only one underlying cause of death.,183 +",Cause of death type. There should be only one underlying cause of death.,183 DEATH_CAUSE,DEATH_CAUSE_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,L;N;D;S;T;NI;UN;OT,"L=Other, locally defined N=National Death Index D=Social Security @@ -723,18 +723,18 @@ OT=Other ","Source of cause of death information. Guidance: ?Other, locally defined? may be used to indicate presence of deaths reported from EHR systems, such as in-patient hospital deaths or dead on arrival. -",184 +",184 DEATH_CAUSE,DEATH_CAUSE_CONFIDENCE,RDBMS Text(2),SAS Char(2),,NO,,E;F;P;NI;UN;OT,"E=Excellent F=Fair P=Poor NI=No information UN=Unknown OT=Other -","Confidence in the accuracy of the cause of death based on source, match, number of reporting sources, discrepancies, etc.",185 -HARVEST,NETWORKID,RDBMS Text(10),SAS Char(10),,NO,,,,This identifier is assigned by DSSNI operations,186 -HARVEST,NETWORK_NAME,RDBMS Text(20),SAS Char(20),,NO,,,,Descriptive name of the network,187 -HARVEST,DATAMARTID,RDBMS Text(10),SAS Char(10),,NO,,,,This identifier is assigned by DSSNI operations,188 -HARVEST,DATAMART_NAME,RDBMS Text(20),SAS Char(20),,NO,,,,Descriptive name of the datamart,189 +","Confidence in the accuracy of the cause of death based on source, match, number of reporting sources, discrepancies, etc.",185 +HARVEST,NETWORKID,RDBMS Text(10),SAS Char(10),,NO,,,,This identifier is assigned by DSSNI operations,186 +HARVEST,NETWORK_NAME,RDBMS Text(20),SAS Char(20),,NO,,,,Descriptive name of the network,187 +HARVEST,DATAMARTID,RDBMS Text(10),SAS Char(10),,NO,,,,This identifier is assigned by DSSNI operations,188 +HARVEST,DATAMART_NAME,RDBMS Text(20),SAS Char(20),,NO,,,,Descriptive name of the datamart,189 HARVEST,DATAMART_PLATFORM,RDBMS Text(2),SAS Char(2),,NO,,"01;02;03;04;05;NI;UN;OT ","01=SQL Server 02=Oracle @@ -744,20 +744,20 @@ HARVEST,DATAMART_PLATFORM,RDBMS Text(2),SAS Char(2),,NO,,"01;02;03;04;05;NI;UN;O NI=No information UN=Unknown OT=Other -",,190 -HARVEST,CDM_VERSION,NUMBER(8),Numeric(8),,NO,,,,"Version currently implemented within this datamart (for example, 1.0, 2.0, 3.0).",191 +",,190 +HARVEST,CDM_VERSION,NUMBER(8),Numeric(8),,NO,,,,"Version currently implemented within this datamart (for example, 1.0, 2.0, 3.0).",191 HARVEST,DATAMART_CLAIMS,RDBMS Text(2),SAS Char(2),,NO,,01;02;NI;UN;OT,"01=Not present 02=Present NI=No information UN=Unknown OT=Other -",Datamart includes claims data source(s),192 +",Datamart includes claims data source(s),192 HARVEST,DATAMART_EHR,RDBMS Text(2),SAS Char(2),,NO,,01;02;NI;UN;OT,"01=Not present 02=Present NI=No information UN=Unknown OT=Other -",Datamart includes EHR data source(s),193 +",Datamart includes EHR data source(s),193 HARVEST,BIRTH_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -767,7 +767,7 @@ OT=Other ","Data management strategy currently present in the BIRTH_DATE field on the DEMOGRAPHIC table Please see notes for additional definitions. -",194 +",194 HARVEST,ENR_START_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -777,7 +777,7 @@ OT=Other ","Data management strategy currently present in the ENR_START_DATE field on the ENROLLMENT table Please see notes for additional definitions. -",195 +",195 HARVEST,ENR_END_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -787,7 +787,7 @@ OT=Other ","Data management strategy currently present in the ENR_END_DATE field on the ENROLLMENT table Please see notes for additional definitions. -",196 +",196 HARVEST,ADMIT_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -797,7 +797,7 @@ OT=Other ","Data management strategy currently present in the ADMIT_DATE field on the ENCOUNTER table Please see notes for additional definitions. -",197 +",197 HARVEST,DISCHARGE_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -807,7 +807,7 @@ OT=Other ","Data management strategy currently present in the DISCHARGE_DATE field on the ENCOUNTER table Please see notes for additional definitions. -",198 +",198 HARVEST,PX_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -817,7 +817,7 @@ OT=Other ","Data management strategy currently present in the PX_DATE field on the PROCEDURES table Please see notes for additional definitions. -",199 +",199 HARVEST,RX_ORDER_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -827,7 +827,7 @@ OT=Other ","Data management strategy currently present in the RX_ORDER_DATE field on the PRESCRIBING table Please see notes for additional definitions. -",200 +",200 HARVEST,RX_START_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -837,7 +837,7 @@ OT=Other ","Data management strategy currently present in the RX_START_DATE field on the PRESCRIBING table Please see notes for additional definitions. -",201 +",201 HARVEST,RX_END_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -847,7 +847,7 @@ OT=Other ","Data management strategy currently present in the RX_END_DATE field on the PRESCRIBING table Please see notes for additional definitions. -",202 +",202 HARVEST,DISPENSE_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -857,7 +857,7 @@ OT=Other ","Data management strategy currently present in the DISPENSE_DATE field on the DISPENSING table Please see notes for additional definitions. -",203 +",203 HARVEST,LAB_ORDER_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -867,7 +867,7 @@ OT=Other ","Data management strategy currently present in the LAB_ORDER_DATE field on the LAB_RESULT_CM table Please see notes for additional definitions. -",204 +",204 HARVEST,SPECIMEN_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -877,7 +877,7 @@ OT=Other ","Data management strategy currently present in the SPECIMEN_DATE field on the LAB_RESULT_CM table Please see notes for additional definitions. -",205 +",205 HARVEST,RESULT_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -887,7 +887,7 @@ OT=Other ","Data management strategy currently present in the RESULT_DATE field on the LAB_RESULT_CM table Please see notes for additional definitions. -",206 +",206 HARVEST,MEASURE_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -897,7 +897,7 @@ OT=Other ","Data management strategy currently present in the MEASURE_DATE field on the VITAL table Please see notes for additional definitions. -",207 +",207 HARVEST,ONSET_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -907,7 +907,7 @@ OT=Other ","Data management strategy currently present in the ONSET_DATE field on the CONDITION table Please see notes for additional definitions. -",208 +",208 HARVEST,REPORT_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -917,7 +917,7 @@ OT=Other ","Data management strategy currently present in the REPORT_DATE field on the CONDITION table Please see notes for additional definitions. -",209 +",209 HARVEST,RESOLVE_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -927,7 +927,7 @@ OT=Other ","Data management strategy currently present in the RESOLVE_DATE field on the CONDITION table Please see notes for additional definitions. -",210 +",210 HARVEST,PRO_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation 02=Imputation for incomplete dates 03=Date obfuscation @@ -937,18 +937,22 @@ OT=Other ","Data management strategy currently present in the PRO_DATE field on the PRO_CM table Please see notes for additional definitions. -",211 -HARVEST,REFRESH_DEMOGRAPHIC_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DEMOGRAPHIC table. This date should be null if the table does not have records.,212 -HARVEST,REFRESH_ENROLLMENT_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the ENROLLMENT table. This date should be null if the table does not have records.,213 -HARVEST,REFRESH_ENCOUNTER_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the ENCOUNTER table. This date should be null if the table does not have records.,214 -HARVEST,REFRESH_DIAGNOSIS_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DIAGNOSIS table. This date should be null if the table does not have records.,215 -HARVEST,REFRESH_PROCEDURES_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PROCEDURES table. This date should be null if the table does not have records.,216 -HARVEST,REFRESH_VITAL_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the VITAL table. This date should be null if the table does not have records.,217 -HARVEST,REFRESH_DISPENSING_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DISPENSING table. This date should be null if the table does not have records.,218 -HARVEST,REFRESH_LAB_RESULT_CM_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the LAB_RESULT_CM table. This date should be null if the table does not have records.,219 -HARVEST,REFRESH_CONDITION_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the CONDITION table. This date should be null if the table does not have records.,220 -HARVEST,REFRESH_PRO_CM_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PRO_CM table. This date should be null if the table does not have records.,221 -HARVEST,REFRESH_PRESCRIBING_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PRESCRIBING table. This date should be null if the table does not have records.,222 -HARVEST,REFRESH_PCORNET_TRIAL_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PCORNET_TRIAL table. This date should be null if the table does not have records.,223 -HARVEST,REFRESH_DEATH_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DEATH table. This date should be null if the table does not have records.,224 -HARVEST,REFRESH_DEATH_CAUSE_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DEATH_CAUSE table. This date should be null if the table does not have records.,225 +",211 +HARVEST,REFRESH_DEMOGRAPHIC_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DEMOGRAPHIC table. This date should be null if the table does not have records.,212 +HARVEST,REFRESH_ENROLLMENT_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the ENROLLMENT table. This date should be null if the table does not have records.,213 +HARVEST,REFRESH_ENCOUNTER_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the ENCOUNTER table. This date should be null if the table does not have records.,214 +HARVEST,REFRESH_DIAGNOSIS_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DIAGNOSIS table. This date should be null if the table does not have records.,215 +HARVEST,REFRESH_PROCEDURES_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PROCEDURES table. This date should be null if the table does not have records.,216 +HARVEST,REFRESH_VITAL_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the VITAL table. This date should be null if the table does not have records.,217 +HARVEST,REFRESH_DISPENSING_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DISPENSING table. This date should be null if the table does not have records.,218 +HARVEST,REFRESH_LAB_RESULT_CM_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the LAB_RESULT_CM table. This date should be null if the table does not have records.,219 +HARVEST,REFRESH_CONDITION_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the CONDITION table. This date should be null if the table does not have records.,220 +HARVEST,REFRESH_PRO_CM_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PRO_CM table. This date should be null if the table does not have records.,221 +HARVEST,REFRESH_PRESCRIBING_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PRESCRIBING table. This date should be null if the table does not have records.,222 +HARVEST,REFRESH_PCORNET_TRIAL_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PCORNET_TRIAL table. This date should be null if the table does not have records.,223 +HARVEST,REFRESH_DEATH_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DEATH table. This date should be null if the table does not have records.,224 +HARVEST,REFRESH_DEATH_CAUSE_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DEATH_CAUSE table. This date should be null if the table does not have records.,225 +MED_ADMIN,,,,,,,,,, +PROVIDER,,,,,,,,,, +OBS_CLIN,,,,,,,,,, +OBS_GEN,,,,,,,,,, From 628c779098d3f37ecf899f06dba275f60a50f82b Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 12 Apr 2018 13:37:07 -0500 Subject: [PATCH 352/507] Create CDM_STATUS table during initialization --- Oracle/pcornet_init.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index ffc9e74..64ea744 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -113,6 +113,20 @@ BEGIN END; / +BEGIN +PMN_DROPSQL('DROP TABLE cdm_status'); +END; +/ + +CREATE TABLE PCORNET_CDM.CDM_STATUS + ( + STATUS VARCHAR2(50 BYTE) NOT NULL ENABLE, + LAST_UPDATE DATE NOT NULL ENABLE, + RECORDS NUMBER(*,0), + GROUP_START NUMBER, + GROUP_END NUMBER + ) +/ BEGIN PMN_DROPSQL('DROP TABLE pcornet_codelist'); From 19cec075d4a0af953aa08e0b79d74d7ab7b31de4 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 13 Apr 2018 11:16:06 -0500 Subject: [PATCH 353/507] Added distinct modifier and comment on death unknown flag. --- Oracle/death.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Oracle/death.sql b/Oracle/death.sql index 757b150..e30a70c 100644 --- a/Oracle/death.sql +++ b/Oracle/death.sql @@ -28,8 +28,10 @@ with death_source_map as ( select 'NAACCR|1760:0' concept_cd, 'T' death_source, null unknown from dual union all select 'NAACCR|1760:4' concept_cd, 'T' death_source, null unknown from dual + -- Possible exception, NAACCR|1760:12 is not handled here as there are no examples in the data. + -- This is the case where date of death is flagged as unknown. ) -select /*+ parallel(0) */ obs.patient_num patid +select distinct /*+ parallel(0) */ obs.patient_num patid , case when dmap.unknown = 1 then DATE '2100-12-31' else obs.start_date end death_date , case when dmap.unknown = 1 then 'OT' else 'N' end death_date_impute , dmap.death_source From fde2e440cc507a23e3702e320bb5542d8041e6ea Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 17 Apr 2018 15:47:11 -0500 Subject: [PATCH 354/507] Added post processing procedure to pcornet_loader --- Oracle/pcornet_loader.sql | 52 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/Oracle/pcornet_loader.sql b/Oracle/pcornet_loader.sql index 48a901b..a181c21 100644 --- a/Oracle/pcornet_loader.sql +++ b/Oracle/pcornet_loader.sql @@ -1,3 +1,55 @@ +/** pcornet_loader - perform post-processing operations. +*/ +create or replace procedure PCORNetPostProc as +begin + + /* Copy providerid from encounter table to diagnosis, procedures tables. + CDM specification says: + "Please note: This is a field replicated from the ENCOUNTER table." + */ + merge into diagnosis d + using encounter e + on (d.encounterid = e.encounterid) + when matched then update set d.providerid = e.providerid; + + merge into procedures p + using encounter e + on (p.encounterid = e.encounterid) + when matched then update set p.providerid = e.providerid; + + merge into prescribing p + using encounter e + on (p.encounterid = e.encounterid) + when matched then update set p.rx_providerid = e.providerid; + + /* Currently in HERON, we have height in cm and weight in oz (from visit vitals). + The CDM wants height in inches and weight in pounds. */ + update vital v set v.ht = v.ht / 2.54; + update vital v set v.wt = v.wt / 16; + + /* Remove rows from the PRESCRIBING table where RX_* fields are null + TODO: Remove this when fixed in HERON + */ + delete + from prescribing + where rx_basis is null + and rx_quantity is null + and rx_frequency is null + and rx_refills is null + ; + + /* Removed bad NDC code which make their way in from the source system + (i.e 00000000000 and 99999999999) */ + delete from dispensing + where ndc in ('00000000000', '99999999999') + ; + +end PCORNetPostProc; +/ +BEGIN +PCORNetPostProc(); +END; +/ insert into cdm_status (status, last_update) values ('pcornet_loader', sysdate ) / select 1 from cdm_status where status = 'pcornet_loader' \ No newline at end of file From 3d186d2823adc94098bf7a10fa9d66413b7e5b83 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 18 Apr 2018 15:37:06 -0500 Subject: [PATCH 355/507] Set default enrollment months to 42. --- i2p_tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index 764b3c4..700d290 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -16,7 +16,7 @@ class CDMScriptTask(SqlScriptTask): def variables(self) -> Environment: return dict(datamart_id='C4UK', datamart_name='University of Kansas', i2b2_data_schema='BLUEHERONDATA', min_pat_list_date_dd_mon_rrrr='01-Jan-2010', min_visit_date_dd_mon_rrrr='01-Jan-2010', - i2b2_meta_schema='BLUEHERONMETADATA', enrollment_months_back='2', network_id='C4', + i2b2_meta_schema='BLUEHERONMETADATA', enrollment_months_back='42', network_id='C4', network_name='GPC', i2b2_etl_schema='HERON_ETL_3') @@ -26,6 +26,7 @@ class condition(CDMScriptTask): def requires(self): return [encounter()] + class death(CDMScriptTask): script = Script.death From 5f8aa560ac8ff99426aa931fe643ac788b9693c5 Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 23 Apr 2018 16:08:57 -0500 Subject: [PATCH 356/507] Change prescribing_transfer to a real table, drop parallel inserts. --- Oracle/prescribing.sql | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index f8b7adf..4ed38ae 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -127,7 +127,7 @@ begin PMN_DROPSQL('DROP TABLE prescribing_transfer'); end; / -create global temporary table prescribing_transfer on commit preserve rows as +create table prescribing_transfer as select * from prescribing where 1 = 0 / @@ -212,7 +212,7 @@ insert /*+ append */ into prescribing_transfer ( PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, RX_QUANTITY, RX_QUANTITY_UNIT, RX_REFILLS, RX_DAYS_SUPPLY, RX_FREQUENCY, RX_BASIS, RAW_RX_MED_NAME, RAW_RXNORM_CUI ) - select /*+ use_nl(freq quantity supply refills) parallel(10) */ + select /*+ use_nl(freq quantity supply refills) */ m.patient_num PATID, m.Encounter_Num ENCOUNTERID, m.provider_id RX_PROVIDERID, @@ -272,7 +272,7 @@ insert /*+ append */ into prescribing_transfer ( commit; -insert /*+ append parallel(10) */ into prescribing ( +insert /*+ append */ into prescribing ( PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, RX_QUANTITY, RX_QUANTITY_UNIT, RX_REFILLS, RX_DAYS_SUPPLY, RX_FREQUENCY, RX_BASIS, RAW_RX_MED_NAME, RAW_RXNORM_CUI ) @@ -292,7 +292,9 @@ BEGIN PCORNetPrescribing(); END; / -truncate table prescribing_transfer +BEGIN +PMN_DROPSQL('DROP TABLE prescribing_transfer'); +END; / insert into cdm_status (status, last_update, records) select 'prescribing', sysdate, count(*) from prescribing / From b28539ccdd08cca211255b560b5974cbf6d8c257 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 26 Apr 2018 09:51:53 -0500 Subject: [PATCH 357/507] Update demographic language and diagnosis pdx --- Oracle/demographic.sql | 12 ++ Oracle/diagnosis.sql | 4 +- Oracle/language.csv | 428 +++++++++++++++++++++++++++++++++++++++++ Oracle/prescribing.sql | 7 +- i2p_tasks.py | 5 + 5 files changed, 451 insertions(+), 5 deletions(-) create mode 100644 Oracle/language.csv diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql index af0866b..e8daee1 100644 --- a/Oracle/demographic.sql +++ b/Oracle/demographic.sql @@ -14,6 +14,7 @@ CREATE TABLE demographic( HISPANIC varchar(2) NULL, BIOBANK_FLAG varchar(1) DEFAULT 'N', RACE varchar(2) NULL, + PAT_PREF_LANGUAGE_SPOKEN varchar(3) NULL, RAW_SEX varchar(50) NULL, RAW_SEXUAL_ORIENTATION varchar(50) NULL, RAW_GENDER_IDENTITY varchar(50) NULL, @@ -187,6 +188,17 @@ FETCH getsql INTO sqltext; END LOOP; CLOSE getsql; +merge into demographic d +using ( + select NVL(code, 'OT') as code, language_cd, patient_num + from language_map + right join i2b2patient on + case when language_cd is NULL then 'no information' else language_cd end = lower(descriptive_text) +) l +on (d.patid = l.patient_num) +when matched then update +set d.PAT_PREF_LANGUAGE_SPOKEN = l.code; + execute immediate 'create unique index demographic_pk on demographic (PATID)'; GATHER_TABLE_STATS('DEMOGRAPHIC'); diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 8fd7a2f..514cfb4 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -209,9 +209,9 @@ select distinct factline.patient_num, factline.encounter_num encounterid, enc_ty , factline.dx_type dxtype, CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END dx_source, nvl(SUBSTR(originsource,INSTR(originsource, ':')+1,2),'NI') dx_origin, - CASE WHEN enc_type in ('EI', 'IP', 'IS') -- PDX is "relevant only on IP and IS encounters" + CASE WHEN enc_type in ('EI', 'IP', 'IS', 'OS') THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, ':')+1,2),'NI') - ELSE 'X' END PDX + ELSE null END PDX from diag_fact_cutoff_filter factline inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num left outer join sourcefact diff --git a/Oracle/language.csv b/Oracle/language.csv new file mode 100644 index 0000000..ec29c0a --- /dev/null +++ b/Oracle/language.csv @@ -0,0 +1,428 @@ +Code,Descriptive_Text +AAR,Afar +ABK,Abkhazian +ACE,Achinese +ACH,Acoli +ADA,Adangme +ADY,Adyghe +ADY,Adygei +AFR,Afrikaans +AIN,Ainu +AKA,Akan +ALE,Aleut +ALT,Southern Altai +AMH,Amharic +ANP,Angika +ARA,Arabic +ARG,Aragonese +ARN,Mapudungun +ARN,Mapuche +ARP,Arapaho +ARW,Arawak +ASM,Assamese +AST,Asturian +AST,Bable +AST,Leonese +AST,Asturleonese +AVA,Avaric +AWA,Awadhi +AYM,Aymara +AZE,Azerbaijani +BAK,Bashkir +BAL,Baluchi +BAM,Bambara +BAN,Balinese +BAS,Basa +BEJ,Beja +BEJ,Bedawiyet +BEL,Belarusian +BEM,Bemba +BEN,Bengali +BHO,Bhojpuri +BIK,Bikol +BIN,Bini +BIN,Edo +BIS,Bislama +BLA,Siksika +BOD,Tibetan +BOS,Bosnian +BRA,Braj +BRE,Breton +BUA,Buriat +BUG,Buginese +BUL,Bulgarian +BYN,Bilin +BYN,Blin +CAD,Caddo +CAR,Galibi Carib +CAT,Catalan +CAT,Valencian +CEB,Cebuano +CES,Czech +CHA,Chamorro +CHE,Chechen +CHK,Chuukese +CHM,Mari +CHN,Chinook jargon +CHO,Choctaw +CHP,Chipewyan +CHP,Dene Suline +CHR,Cherokee +CHV,Chuvash +CHY,Cheyenne +COR,Cornish +COS,Corsican +CRE,Cree +CRH,Crimean Tatar +CRH,Crimean Turkish +CSB,Kashubian +CYM,Welsh +DAK,Dakota +DAN,Danish +DAR,Dargwa +DEL,Delaware +DEN,Slave (Athapascan) +DEU,German +DGR,Dogrib +DIN,Dinka +DIV,Dhivehi +DIV,Dhivehi +DIV,Maldivian +DOI,Dogri +DSB,Lower Sorbian +DUA,Duala +DYU,Dyula +DZO,Dzongkha +EFI,Efik +EKA,Ekajuk +ELL,Modern Greek (1453–) +ENG,English +EST,Estonian +EUS,Basque +EWE,Ewe +EWO,Ewondo +FAN,Fang +FAO,Faroese +FAS,Persian +FAT,Fanti +FIJ,Fijian +FIL,Filipino +FIL,Pilipino +FIN,Finnish +FON,Fon +FRA,French +FRR,Northern Frisian +FRS,Eastern Frisian +FRY,Western Frisian +FUL,Fulah +FUR,Friulian +GAA,Ga +GAY,Gayo +GBA,Gbaya +GIL,Gilbertese +GLA,Gaelic +GLA,Scottish Gaelic +GLE,Irish +GLG,Galician +GLV,Manx +GON,Gondi +GOR,Gorontalo +GRB,Grebo +GRN,Guarani +GSW,Swiss German +GSW,Alemannic +GSW,Alsatian +GUJ,Gujarati +GWI,Gwich?in +HAI,Haida +HAT,Haitian +HAT,Haitian Creole +HAU,Hausa +HAW,Hawaiian +HEB,Hebrew +HER,Herero +HIL,Hiligaynon +HIN,Hindi +HMN,Hmong +HMN,Mong +HMO,Hiri Motu +HRV,Croatian +HSB,Upper Sorbian +HUN,Hungarian +HUP,Hupa +HYE,Armenian +IBA,Iban +IBO,Igbo +III,Sichuan Yi +III,Nuosu +IKU,Inuktitut +ILO,Iloko +IND,Indonesian +INH,Ingush +IPK,Inupiaq +ISL,Icelandic +ITA,Italian +JAV,Javanese +JPN,Japanese +JPR,Judeo-Persian +JRB,Judeo-Arabic +KAA,Kara-Kalpak +KAB,Kabyle +KAC,Kachin +KAC,Jingpho +KAL,Kalaallisut +KAL,Greenlandic +KAM,Kamba +KAN,Kannada +KAS,Kashmiri +KAT,Georgian +KAU,Kanuri +KAZ,Kazakh +KBD,Kabardian +KHA,Khasi +KHM,Central Khmer +KIK,Kikuyu +KIK,Gikuyu +KIN,Kinyarwanda +KIR,Kirghiz +KIR,Kyrgyz +KMB,Kimbundu +KOK,Konkani +KOM,Komi +KON,Kongo +KOR,Korean +KOS,Kosraean +KPE,Kpelle +KRC,Karachay-Balkar +KRL,Karelian +KRU,Kurukh +KUA,Kuanyama +KUA,Kwanyama +KUM,Kumyk +KUR,Kurdish +KUT,Kutenai +LAD,Ladino +LAH,Lahnda +LAM,Lamba +LAO,Lao +LAV,Latvian +LEZ,Lezghian +LIM,Limburgan +LIM,Limburger +LIM,Limburgish +LIN,Lingala +LIT,Lithuanian +LOL,Mongo +LOZ,Lozi +LTZ,Luxembourgish +LTZ,Letzeburgesch +LUA,Luba-Lulua +LUB,Luba-Katanga +LUG,Ganda +LUI,Luiseno +LUN,Lunda +LUO,Luo (Kenya and Tanzania) +LUS,Lushai +MAD,Madurese +MAG,Magahi +MAH,Marshallese +MAI,Maithili +MAK,Makasar +MAL,Malayalam +MAN,Mandingo +MAR,Marathi +MAS,Masai +MDF,Moksha +MDR,Mandar +MEN,Mende +MIC,Mi'kmaq +MIC,Micmac +MIN,Minangkabau +MKD,Macedonian +MLG,Malagasy +MLT,Maltese +MNC,Manchu +MNI,Manipuri +MOH,Mohawk +MON,Mongolian +MOS,Mossi +MRI,Maori +MSA,Malay +MUS,Creek +MWL,Mirandese +MWR,Marwari +MYA,Burmese +MYV,Erzya +NAP,Neapolitan +NAU,Nauru +NAV,Navajo +NAV,Navaho +NBL,South Ndebele +NDE,North Ndebele +NDO,Ndonga +NDS,Low German +NDS,Low Saxon +NEP,Nepali +NEW,Nepal Bhasa +NEW,Newari +NIA,Nias +NIU,Niuean +NLD,Dutch +NLD,Flemish +NNO,Norwegian Nynorsk +NOB,Norwegian Bokmål +NOG,Nogai +NOR,Norwegian +NQO,N'Ko +NSO,Pedi +NSO,Sepedi +NSO,Northern Sotho +NYA,Chichewa +NYA,Chewa +NYA,Nyanja +NYM,Nyamwezi +NYN,Nyankole +NYO,Nyoro +NZI,Nzima +OCI,Occitan (post 1500) +OJI,Ojibwa +ORI,Oriya +ORM,Oromo +OSA,Osage +OSS,Ossetian +OSS,Ossetic +PAG,Pangasinan +PAM,Pampanga +PAM,Kapampangan +PAN,Panjabi +PAN,Punjabi +PAP,Papiamento +PAU,Palauan +POL,Polish +PON,Pohnpeian +POR,Portuguese +PUS,Pushto +PUS,Pashto +QUE,Quechua +RAJ,Rajasthani +RAP,Rapanui +RAR,Rarotongan +RAR,Cook Islands Maori +ROH,Romansh +ROM,Romany +RON,Romanian +RON,Moldavian +RON,Moldovan +RUN,Rundi +RUP,Aromanian +RUP,Arumanian +RUP,Macedo-Romanian +RUS,Russian +SAD,Sandawe +SAG,Sango +SAH,Yakut +SAS,Sasak +SAT,Santali +SCN,Sicilian +SCO,Scots +SEL,Selkup +SHN,Shan +SID,Sidamo +SIN,Sinhala +SIN,Sinhalese +SLK,Slovak +SLV,Slovenian +SMA,Southern Sami +SME,Northern Sami +SMJ,Lule Sami +SMN,Inari Sami +SMO,Samoan +SMS,Skolt Sami +SNA,Shona +SND,Sindhi +SNK,Soninke +SOM,Somali +SOT,Southern Sotho +SPA,Spanish +SPA,Castilian +SQI,Albanian +SRD,Sardinian +SRN,Sranan Tongo +SRP,Serbian +SRR,Serer +SSW,Swati +SUK,Sukuma +SUN,Sundanese +SUS,Susu +SWA,Swahili +SWE,Swedish +SYR,Syriac +TAH,Tahitian +TAM,Tamil +TAT,Tatar +TEL,Telugu +TEM,Timne +TER,Tereno +TET,Tetum +TGK,Tajik +TGL,Tagalog +THA,Thai +TIG,Tigre +TIR,Tigrinya +TIV,Tiv +TKL,Tokelau +TLI,Tlingit +TMH,Tamashek +TOG,Tonga (Nyasa) +TON,Tonga (Tonga Islands) +TPI,Tok Pisin +TSI,Tsimshian +TSN,Tswana +TSO,Tsonga +TUK,Turkmen +TUM,Tumbuka +TUR,Turkish +TVL,Tuvalua +TWI,Twi +TYV,Tuvinian +UDM,Udmurt +UIG,Uighur +UIG,Uyghur +UKR,Ukrainian +UMB,Umbundu +URD,Urdu +UZB,Uzbek +VAI,Vai +VEN,Venda +VIE,Vietnamese +VOT,Votic +WAL,Wolaitta +WAL,Wolaytta +WAR,Waray +WAS,Washo +WLN,Walloon +WOL,Wolof +XAL,Kalmyk +XAL,Oirat +XHO,Xhosa +YAO,Yao +YAP,Yapese +YID,Yiddish +YOR,Yoruba +ZAP,Zapotec +ZEN,Zenaga +ZGH,Standard Moroccan Tamazight +ZHA,Zhuang +ZHA,Chuang +ZHO,Chinese +ZUL,Zulu +ZUN,Zuni +ZZA,Zaza +ZZA,Dimili +ZZA,Dimli +ZZA,Kirdki +ZZA,Kirmanjki +ZZA,Zazaki +NI,No information +UN,Unknown +OT,Other diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 4ed38ae..37c2576 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -149,9 +149,10 @@ execute immediate 'truncate table refills'; execute immediate 'truncate table supply'; insert into basis -select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact basis - inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num - join pcornet_med basiscode +select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd +from i2b2medfact basis +inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num +join pcornet_med basiscode on basis.modifier_cd = basiscode.c_basecode and basiscode.c_fullname like '\PCORI_MOD\RX_BASIS\%'; diff --git a/i2p_tasks.py b/i2p_tasks.py index 700d290..5069c00 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -193,3 +193,8 @@ class loadLabNormal(LoadCSV): class loadHarvestLocal(LoadCSV): tablename = 'HARVEST_LOCAL' csvname = 'Oracle/harvest_local.csv' + + +class loadLanguage(LoadCSV): + tablename = 'LANGUAGE_MAP' + csvname = 'Oracle/language.csv' From 70a5f22c0de66a44566b68cc2d3106595775ff85 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 26 Apr 2018 10:02:40 -0500 Subject: [PATCH 358/507] Fix utf-8 decode error --- Oracle/language.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/language.csv b/Oracle/language.csv index ec29c0a..1f2fb3d 100644 --- a/Oracle/language.csv +++ b/Oracle/language.csv @@ -133,7 +133,7 @@ GSW,Swiss German GSW,Alemannic GSW,Alsatian GUJ,Gujarati -GWI,Gwich?in +GWI,Gwich'in HAI,Haida HAT,Haitian HAT,Haitian Creole From 77d78bdab6c7950d53b7ac6e1f432f8ca78700ef Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 26 Apr 2018 10:11:56 -0500 Subject: [PATCH 359/507] Fix utf-8 decode errors --- Oracle/language.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/language.csv b/Oracle/language.csv index 1f2fb3d..1116e38 100644 --- a/Oracle/language.csv +++ b/Oracle/language.csv @@ -95,7 +95,7 @@ DYU,Dyula DZO,Dzongkha EFI,Efik EKA,Ekajuk -ELL,Modern Greek (1453–) +ELL,Modern Greek (1453-) ENG,English EST,Estonian EUS,Basque @@ -270,7 +270,7 @@ NIU,Niuean NLD,Dutch NLD,Flemish NNO,Norwegian Nynorsk -NOB,Norwegian Bokmål +NOB,Norwegian Bokmal NOG,Nogai NOR,Norwegian NQO,N'Ko @@ -284,7 +284,7 @@ NYM,Nyamwezi NYN,Nyankole NYO,Nyoro NZI,Nzima -OCI,Occitan (post 1500) +OCI,Occitan (post 1500) OJI,Ojibwa ORI,Oriya ORM,Oromo From 3eb6c9e1dfd4b82d2b2a450235de5cdf7f9d262b Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 26 Apr 2018 10:47:13 -0500 Subject: [PATCH 360/507] Capitalizing table column names --- Oracle/language.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/language.csv b/Oracle/language.csv index 1116e38..945c5c7 100644 --- a/Oracle/language.csv +++ b/Oracle/language.csv @@ -1,4 +1,4 @@ -Code,Descriptive_Text +CODE,DESCRIPTIVE_TEXT AAR,Afar ABK,Abkhazian ACE,Achinese From 4228a855262b9b2c02f64f91dc32fff4b1afca53 Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 30 Apr 2018 16:24:30 -0500 Subject: [PATCH 361/507] Optimized prescribing build --- Oracle/pcornet_loader.sql | 4 + Oracle/prescribing.sql | 456 ++++++++++++++++---------------------- i2p_tasks.py | 48 +++- 3 files changed, 243 insertions(+), 265 deletions(-) diff --git a/Oracle/pcornet_loader.sql b/Oracle/pcornet_loader.sql index a181c21..10fdd6c 100644 --- a/Oracle/pcornet_loader.sql +++ b/Oracle/pcornet_loader.sql @@ -22,6 +22,10 @@ begin on (p.encounterid = e.encounterid) when matched then update set p.rx_providerid = e.providerid; + update pcornet_cdm.prescribing + set rx_providerid = null where + rx_providerid = '@'; + /* Currently in HERON, we have height in cm and weight in oz (from visit vitals). The CDM wants height in inches and weight in pounds. */ update vital v set v.ht = v.ht / 2.54; diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 37c2576..08cf7d4 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -1,302 +1,232 @@ /** prescribing - create and populate the prescribing table. */ + BEGIN PMN_DROPSQL('DROP TABLE prescribing'); END; / -CREATE TABLE prescribing( - PRESCRIBINGID varchar(19) primary key, - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NULL, - RX_PROVIDERID varchar(50) NULL, -- NOTE: The spec has a _ before the ID, but this is inconsistent. - RX_ORDER_DATE date NULL, - RX_ORDER_TIME varchar (5) NULL, - RX_START_DATE date NULL, - RX_END_DATE date NULL, - RX_QUANTITY number(18,5) NULL, - RX_QUANTITY_UNIT varchar(2) NULL, - RX_REFILLS number(18,5) NULL, - RX_DAYS_SUPPLY number (18,5) NULL, - RX_FREQUENCY varchar(2) NULL, - RX_BASIS varchar (2) NULL, - RXNORM_CUI varchar(8) NULL, - RAW_RX_MED_NAME varchar (50) NULL, - RAW_RX_FREQUENCY varchar (50) NULL, - RAW_RX_QUANTITY varchar(50) NULL, - RAW_RX_NDC varchar(50) NULL, - RAW_RXNORM_CUI varchar (50) NULL -) -/ - BEGIN -PMN_DROPSQL('DROP sequence prescribing_seq'); +PMN_DROPSQL('drop table prescribing_key'); END; / -create sequence prescribing_seq -/ - -create or replace trigger prescribing_trg -before insert on prescribing -for each row -begin - select prescribing_seq.nextval into :new.PRESCRIBINGID from dual; -end; -/ - BEGIN -PMN_DROPSQL('DROP TABLE basis'); +PMN_DROPSQL('drop table prescribing_w_cui'); END; / - -CREATE TABLE BASIS ( - PCORI_BASECODE VARCHAR2(50) NULL, - C_FULLNAME VARCHAR2(700) NOT NULL, - INSTANCE_NUM NUMBER(18) NOT NULL, - START_DATE DATE NOT NULL, - PROVIDER_ID VARCHAR2(50) NOT NULL, - CONCEPT_CD VARCHAR2(50) NOT NULL, - ENCOUNTER_NUM NUMBER(38) NOT NULL, - MODIFIER_CD VARCHAR2(100) NOT NULL - ) -/ - BEGIN -PMN_DROPSQL('DROP TABLE freq'); +PMN_DROPSQL('drop table prescribing_w_refills'); END; / - -CREATE TABLE FREQ ( - PCORI_BASECODE VARCHAR2(50) NULL, - INSTANCE_NUM NUMBER(18) NOT NULL, - START_DATE DATE NOT NULL, - PROVIDER_ID VARCHAR2(50) NOT NULL, - CONCEPT_CD VARCHAR2(50) NOT NULL, - ENCOUNTER_NUM NUMBER(38) NOT NULL, - MODIFIER_CD VARCHAR2(100) NOT NULL - ) -/ - BEGIN -PMN_DROPSQL('DROP TABLE quantity'); +PMN_DROPSQL('drop table prescribing_w_freq'); END; / - -CREATE TABLE QUANTITY ( - NVAL_NUM NUMBER(18,5) NULL, - INSTANCE_NUM NUMBER(18) NOT NULL, - START_DATE DATE NOT NULL, - PROVIDER_ID VARCHAR2(50) NOT NULL, - CONCEPT_CD VARCHAR2(50) NOT NULL, - ENCOUNTER_NUM NUMBER(38) NOT NULL, - MODIFIER_CD VARCHAR2(100) NOT NULL - ) +BEGIN +PMN_DROPSQL('drop table prescribing_w_quantity'); +END; / - BEGIN -PMN_DROPSQL('DROP TABLE refills'); +PMN_DROPSQL('drop table prescribing_w_supply'); END; / - -CREATE TABLE REFILLS ( - NVAL_NUM NUMBER(18,5) NULL, - INSTANCE_NUM NUMBER(18) NOT NULL, - START_DATE DATE NOT NULL, - PROVIDER_ID VARCHAR2(50) NOT NULL, - CONCEPT_CD VARCHAR2(50) NOT NULL, - ENCOUNTER_NUM NUMBER(38) NOT NULL, - MODIFIER_CD VARCHAR2(100) NOT NULL - ) +BEGIN +PMN_DROPSQL('drop table prescribing_w_basis'); +END; / - BEGIN -PMN_DROPSQL('DROP TABLE supply'); +PMN_DROPSQL('DROP sequence prescribing_seq'); END; / -CREATE TABLE SUPPLY ( - NVAL_NUM NUMBER(18,5) NULL, - INSTANCE_NUM NUMBER(18) NOT NULL, - START_DATE DATE NOT NULL, - PROVIDER_ID VARCHAR2(50) NOT NULL, - CONCEPT_CD VARCHAR2(50) NOT NULL, - ENCOUNTER_NUM NUMBER(38) NOT NULL, - MODIFIER_CD VARCHAR2(100) NOT NULL - ) -/ -begin -PMN_DROPSQL('DROP TABLE prescribing_transfer'); -end; -/ -create table prescribing_transfer as -select * from prescribing where 1 = 0 +create sequence prescribing_seq cache 2000 +/ + +/** prescribing_key -- one row per order (inpatient or outpatient) + +Design note on modifiers: +create or replace view rx_mod_design as + select c_basecode + from blueheronmetadata.pcornet_med + -- TODO: fix pcornet_mapping.csv + where c_fullname like '\PCORI_MOD\RX_BASIS\PR\_%' + and c_basecode not in ('MedObs:PRN', -- that's a supplementary flag, not a basis + 'MedObs:Other' -- that's as opposed to Historical; not same category as Inpatient, Outpatient + ) + and c_basecode not like 'RX_BASIS:%' +; + +select case when ( + select listagg(c_basecode, ',') within group (order by c_basecode) modifiers + from rx_mod_design + ) = 'MedObs:Inpatient,MedObs:Outpatient' +then 1 -- pass +else 1/0 -- fail +end modifiers_as_expected +from dual +; +*/ +create table prescribing_key as +select prescribing_seq.nextval prescribingid +, instance_num +, patient_num +, encounter_num +, provider_id +, start_date +, end_date +, concept_cd +, modifier_cd +from blueherondata.observation_fact rx +where rx.modifier_cd in ('MedObs:Inpatient', 'MedObs:Outpatient') +and rx.START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') / -create or replace procedure PCORNetPrescribing as -begin - -PMN_DROPSQL('drop index prescribing_idx'); -PMN_DROPSQL('drop index basis_idx'); -PMN_DROPSQL('drop index freq_idx'); -PMN_DROPSQL('drop index quantity_idx'); -PMN_DROPSQL('drop index refills_idx'); -PMN_DROPSQL('drop index supply_idx'); - -execute immediate 'truncate table prescribing'; -execute immediate 'truncate table basis'; -execute immediate 'truncate table freq'; -execute immediate 'truncate table quantity'; -execute immediate 'truncate table refills'; -execute immediate 'truncate table supply'; - -insert into basis -select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd -from i2b2medfact basis -inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num -join pcornet_med basiscode - on basis.modifier_cd = basiscode.c_basecode - and basiscode.c_fullname like '\PCORI_MOD\RX_BASIS\%'; - -commit; - -execute immediate 'create index basis_idx on basis (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; -GATHER_TABLE_STATS('BASIS'); - -insert into freq -select pcori_basecode,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact freq - inner join encounter enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num - join pcornet_med freqcode - on freq.modifier_cd = freqcode.c_basecode - and freqcode.c_fullname like '\PCORI_MOD\RX_FREQUENCY\%'; - -commit; - -execute immediate 'create index freq_idx on freq (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; -GATHER_TABLE_STATS('FREQ'); - -insert into quantity -select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact quantity - inner join encounter enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num - join pcornet_med quantitycode - on quantity.modifier_cd = quantitycode.c_basecode - and quantitycode.c_fullname like '\PCORI_MOD\RX_QUANTITY\'; - -commit; - -execute immediate 'create index quantity_idx on quantity (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; -GATHER_TABLE_STATS('QUANTITY'); - -insert into refills -select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact refills - inner join encounter enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num - join pcornet_med refillscode - on refills.modifier_cd = refillscode.c_basecode - and refillscode.c_fullname like '\PCORI_MOD\RX_REFILLS\'; - -commit; - -execute immediate 'create index refills_idx on refills (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; -GATHER_TABLE_STATS('REFILLS'); - -insert into supply -select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact supply - inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num - join pcornet_med supplycode - on supply.modifier_cd = supplycode.c_basecode - and supplycode.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\'; - -commit; - -execute immediate 'create index supply_idx on supply (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; -GATHER_TABLE_STATS('SUPPLY'); - -insert /*+ append */ into prescribing_transfer ( - PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, - RX_QUANTITY, RX_QUANTITY_UNIT, RX_REFILLS, RX_DAYS_SUPPLY, RX_FREQUENCY, RX_BASIS, RAW_RX_MED_NAME, RAW_RXNORM_CUI -) - select /*+ use_nl(freq quantity supply refills) */ - m.patient_num PATID, - m.Encounter_Num ENCOUNTERID, - m.provider_id RX_PROVIDERID, - m.start_date RX_ORDER_DATE, - to_char(m.start_date,'HH24:MI') RX_ORDER_TIME, - m.start_date RX_START_DATE, - m.end_date RX_END_DATE, - mo.pcori_cui RXNORM_CUI, - quantity.nval_num RX_QUANTITY, - 'NI' RX_QUANTITY_UNIT, - refills.nval_num RX_REFILLS, - supply.nval_num RX_DAYS_SUPPLY, - substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) RX_FREQUENCY, - substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) RX_BASIS, - substr(mo.c_name, 1, 50) RAW_RX_MED_NAME, - substr(mo.c_basecode, 1, 50) RAW_RXNORM_CUI - - from i2b2medfact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode - inner join encounter enc on enc.encounterid = m.encounter_Num - - left join basis - on m.encounter_num = basis.encounter_num - and m.concept_cd = basis.concept_Cd - and m.start_date = basis.start_date - and m.provider_id = basis.provider_id - and m.instance_num = basis.instance_num - - left join freq - on m.encounter_num = freq.encounter_num - and m.concept_cd = freq.concept_Cd - and m.start_date = freq.start_date - and m.provider_id = freq.provider_id - and m.instance_num = freq.instance_num - - left join quantity - on m.encounter_num = quantity.encounter_num - and m.concept_cd = quantity.concept_Cd - and m.start_date = quantity.start_date - and m.provider_id = quantity.provider_id - and m.instance_num = quantity.instance_num - - left join refills - on m.encounter_num = refills.encounter_num - and m.concept_cd = refills.concept_Cd - and m.start_date = refills.start_date - and m.provider_id = refills.provider_id - and m.instance_num = refills.instance_num - - left join supply - on m.encounter_num = supply.encounter_num - and m.concept_cd = supply.concept_Cd - and m.start_date = supply.start_date - and m.provider_id = supply.provider_id - and m.instance_num = supply.instance_num - - where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); - -commit; +alter table prescribing_key modify (provider_id null) +/ -insert /*+ append */ into prescribing ( - PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, - RX_QUANTITY, RX_QUANTITY_UNIT, RX_REFILLS, RX_DAYS_SUPPLY, RX_FREQUENCY, RX_BASIS, RAW_RX_MED_NAME, RAW_RXNORM_CUI -) -select distinct - PATID, ENCOUNTERID, RX_PROVIDERID, RX_ORDER_DATE, RX_ORDER_TIME, RX_START_DATE, RX_END_DATE, RXNORM_CUI, - RX_QUANTITY, RX_QUANTITY_UNIT, RX_REFILLS, RX_DAYS_SUPPLY, RX_FREQUENCY, RX_BASIS, RAW_RX_MED_NAME, RAW_RXNORM_CUI -from prescribing_transfer; +alter table prescribing_key modify (encounter_num null) +/ -commit; +/** prescribing_w_cui -execute immediate 'create index prescribing_idx on prescribing (PATID, ENCOUNTERID)'; -GATHER_TABLE_STATS('PRESCRIBING'); +take care with cardinalities... -end PCORNetPrescribing; -/ -BEGIN -PCORNetPrescribing(); -END; +select count(distinct c_name), c_basecode +from pcornet_med +group by c_basecode, c_name +having count(distinct c_name) > 1 +order by c_basecode +; +*/ +create table prescribing_w_cui as +select rx.* +, mo.pcori_cui rxnorm_cui +, substr(mo.c_basecode, 1, 50) raw_rxnorm_cui +, substr(mo.c_name, 1, 50) raw_rx_med_name +from prescribing_key rx +left join + (select distinct c_basecode + , c_name + , pcori_cui + from pcornet_med + ) mo +on rx.concept_cd = mo.c_basecode +/ + +/** prescribing_w_refills +This join isn't guaranteed not to introduce more rows, +but at least one measurement showed it does not. + */ +create table prescribing_w_refills as +select rx.* +, refills.nval_num rx_refills +from prescribing_w_cui rx +left join + (select instance_num + , concept_cd + , nval_num + from blueherondata.observation_fact + where modifier_cd = 'RX_REFILLS' + /* aka: + select c_basecode from pcornet_med refillscode + where refillscode.c_fullname like '\PCORI_MOD\RX_REFILLS\' */ + ) refills on refills.instance_num = rx.instance_num and refills.concept_cd = rx.concept_cd +/ + +create table prescribing_w_freq as +select rx.* +, substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) rx_frequency +from prescribing_w_refills rx +left join + (select instance_num + , concept_cd + , pcori_basecode + from blueherondata.observation_fact + join pcornet_med on modifier_cd = c_basecode + and c_fullname like '\PCORI_MOD\RX_FREQUENCY\%' + ) freq on freq.instance_num = rx.instance_num and freq.concept_cd = rx.concept_cd +/ + +create table prescribing_w_quantity as +select rx.* +, quantity.nval_num rx_quantity +from prescribing_w_freq rx +left join + (select instance_num + , concept_cd + , nval_num + from blueherondata.observation_fact + where modifier_cd = 'RX_QUANTITY' + /* aka: + select c_basecode from pcornet_med refillscode + where refillscode.c_fullname like '\PCORI_MOD\RX_QUANTITY\' */ + ) quantity on quantity.instance_num = rx.instance_num and quantity.concept_cd = rx.concept_cd +/ + +create table prescribing_w_supply as +select rx.* +, supply.nval_num rx_days_supply +from prescribing_w_quantity rx +left join + (select instance_num + , concept_cd + , nval_num + from blueherondata.observation_fact + where modifier_cd = 'RX_DAYS_SUPPLY' + /* aka: + select c_basecode from pcornet_med refillscode + where refillscode.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\' */ + ) supply on supply.instance_num = rx.instance_num and supply.concept_cd = rx.concept_cd +/ + +create table prescribing_w_basis as +select rx.* +, substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) rx_basis +from prescribing_w_supply rx +left join + (select instance_num + , concept_cd + , pcori_basecode + from blueherondata.observation_fact + join pcornet_med on modifier_cd = c_basecode + and c_fullname like '\PCORI_MOD\RX_BASIS\%' + and modifier_cd in ('MedObs:Inpatient', 'MedObs:Outpatient') + ) basis on basis.instance_num = rx.instance_num and basis.concept_cd = rx.concept_cd +/ + +create table prescribing as +select rx.prescribingid +, rx.patient_num patid +, rx.encounter_num encounterid +, rx.provider_id rx_providerid +, trunc(rx.start_date) rx_order_date +, to_char(rx.start_date, 'HH24:MI') rx_order_time +, trunc(rx.start_date) rx_start_date +, trunc(rx.end_date) rx_end_date +, rx.rx_quantity +, 'NI' rx_quantity_unit +, rx.rx_refills +, rx.rx_days_supply +, rx.rx_frequency +, decode(rx.modifier_cd, 'MedObs:Inpatient', '01', 'MedObs:Outpatient', '02') rx_basis +, rx.rxnorm_cui +, rx.raw_rx_med_name +, cast(null as varchar(50)) raw_rx_frequency +, cast(null as varchar(50)) raw_rx_quantity +, cast(null as varchar(50)) raw_rx_ndc +, rx.raw_rxnorm_cui +/* ISSUE: HERON should have an actual order time. + idea: store real difference between order date start data, possibly using the update date */ +from prescribing_w_basis rx +/ + +create index prescribing_idx on prescribing (PATID, ENCOUNTERID) / BEGIN -PMN_DROPSQL('DROP TABLE prescribing_transfer'); +GATHER_TABLE_STATS('PRESCRIBING'); END; / + insert into cdm_status (status, last_update, records) select 'prescribing', sysdate, count(*) from prescribing / -select 1 from cdm_status where status = 'prescribing' \ No newline at end of file + +select 1 from cdm_status where status = 'prescribing' diff --git a/i2p_tasks.py b/i2p_tasks.py index 5069c00..5cf8081 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -8,7 +8,9 @@ from sql_syntax import Environment from sqlalchemy.engine import RowProxy from sqlalchemy.exc import DatabaseError -from typing import List +from typing import cast, List, Type + +import luigi class CDMScriptTask(SqlScriptTask): @@ -84,6 +86,7 @@ def requires(self): lab_result_cm(), med_admin(), obs_clin(), obs_gen(), pcornet_trial(), prescribing(), pro_cm(), procedures(), provider(), vital()] + class lab_result_cm(CDMScriptTask): script = Script.lab_result_cm @@ -106,11 +109,52 @@ class obs_gen(CDMScriptTask): script = Script.obs_gen +class CDMPatientGroupTask(CDMScriptTask): + patient_num_first = IntParam() + patient_num_last = IntParam() + patient_num_qty = IntParam(significant=False, default=-1) + group_num = IntParam(significant=False, default=-1) + group_qty = IntParam(significant=False, default=-1) + + def run(self) -> None: + SqlScriptTask.run_bound(self, script_params=dict( + patient_num_first=self.patient_num_first, patient_num_last=self.patient_num_last)) + + +class _PatientNumGrouped(luigi.WrapperTask): + group_tasks = cast(List[Type[CDMPatientGroupTask]], []) # abstract + + def requires(self) -> List[luigi.Task]: + deps = [] # type: List[luigi.Task] + for group_task in self.group_tasks: + survey = patient_chunks_survey() + deps += [survey] + results = survey.results() + if results: + deps += [ + group_task( + group_num=ntile.chunk_num, + group_qty=len(results), + patient_num_qty=ntile.patient_num_qty, + patient_num_first=ntile.patient_num_first, + patient_num_last=ntile.patient_num_last) + for ntile in results + ] + return deps + + class patient_chunks_survey(SqlScriptTask): script = Script.patient_chunks_survey - patient_chunks = IntParam(default=200) + patient_chunks = IntParam(default=20) patient_chunk_max = IntParam(default=None) + @property + def variables(self) -> Environment: + return dict(chunk_qty=str(self.patient_chunks)) + + def run(self) -> None: + SqlScriptTask.run_bound(self, script_params=dict(chunk_qty=str(self.patient_chunks))) + def results(self) -> List[RowProxy]: with self.connection(event='survey results') as lc: q = ''' From 4271b1c8aeb3ebd528d0354a7d5a79aee1d87766 Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 1 May 2018 14:29:16 -0500 Subject: [PATCH 362/507] Fixing EDC data model error. --- Oracle/prescribing.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 08cf7d4..20d883c 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -66,18 +66,18 @@ from dual ; */ create table prescribing_key as -select prescribing_seq.nextval prescribingid +select cast(prescribing_seq.nextval as varchar(19)) prescribingid , instance_num -, patient_num -, encounter_num +, cast(patient_num as varchar(50)) patient_num +, cast(encounter_num as varchar(50)) encounter_num , provider_id , start_date , end_date , concept_cd , modifier_cd from blueherondata.observation_fact rx +join encounter en on rx.encounter_num = en.encounterid where rx.modifier_cd in ('MedObs:Inpatient', 'MedObs:Outpatient') -and rx.START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') / alter table prescribing_key modify (provider_id null) From 521fc83bc3a0841eea085d3b02f331903068c94c Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 1 May 2018 14:33:26 -0500 Subject: [PATCH 363/507] Fixing EDC data model errors. --- Oracle/prescribing.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 20d883c..5b1309b 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -80,10 +80,10 @@ join encounter en on rx.encounter_num = en.encounterid where rx.modifier_cd in ('MedObs:Inpatient', 'MedObs:Outpatient') / -alter table prescribing_key modify (provider_id null) +--alter table prescribing_key modify (provider_id null) / -alter table prescribing_key modify (encounter_num null) +--alter table prescribing_key modify (encounter_num null) / /** prescribing_w_cui From 4e25f1158e3f1b724cbed0c6fa96fc9ed44bc21d Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 1 May 2018 14:34:45 -0500 Subject: [PATCH 364/507] Fixing EDC data model errors. --- Oracle/prescribing.sql | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 5b1309b..894f3aa 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -80,12 +80,6 @@ join encounter en on rx.encounter_num = en.encounterid where rx.modifier_cd in ('MedObs:Inpatient', 'MedObs:Outpatient') / ---alter table prescribing_key modify (provider_id null) -/ - ---alter table prescribing_key modify (encounter_num null) -/ - /** prescribing_w_cui take care with cardinalities... From 7dadd9a9b7c12027f1f4b748a46b20a5eddd187d Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 1 May 2018 14:38:31 -0500 Subject: [PATCH 365/507] Fixing EDC data model errors. --- Oracle/prescribing.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 894f3aa..5d9f1f2 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -80,6 +80,9 @@ join encounter en on rx.encounter_num = en.encounterid where rx.modifier_cd in ('MedObs:Inpatient', 'MedObs:Outpatient') / +alter table prescribing_key modify (provider_id null) +/ + /** prescribing_w_cui take care with cardinalities... From c182261639a47dcbc0aef7d9470974f0f51b628a Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 7 May 2018 15:18:19 -0500 Subject: [PATCH 366/507] script_lib: restore doctests to working order - prune dead flowsheets code ported from heron - disable some tests - use case-insensitive match to look for create table --- Oracle/epic_facts_load.sql | 92 ----- Oracle/epic_flowsheets_transform.sql | 577 --------------------------- epic_flowsheets.py | 192 --------- script_lib.py | 49 +-- sql_syntax.py | 4 +- 5 files changed, 12 insertions(+), 902 deletions(-) delete mode 100644 Oracle/epic_facts_load.sql delete mode 100644 Oracle/epic_flowsheets_transform.sql delete mode 100644 epic_flowsheets.py diff --git a/Oracle/epic_facts_load.sql b/Oracle/epic_facts_load.sql deleted file mode 100644 index 6bff120..0000000 --- a/Oracle/epic_facts_load.sql +++ /dev/null @@ -1,92 +0,0 @@ -/** epic_facts_load.sql - Load observations from Epic Clarity. - -Copyright (c) 2012-2017 University of Kansas Medical Center -part of the HERON* open source codebase; see NOTICE file for license details. -* http://informatics.kumc.edu/work/wiki/HERON - -see also epic_dimensions_load.sql, - http://informatics.kumc.edu/work/wiki/HeronLoad -*/ - -/* Check for id repository, uploader service tables */ -select * from NightHeronData.observation_fact where 1 = 0; -select * from NightHeronData.upload_status where 1 = 0; - -/* We're wasting our time if an upload_status record isn't in place. */ -select case count(*) when 1 then 1 else 1/0 end as upload_status_exists from ( -select * from NightHeronData.upload_status up -where up.upload_id = :upload_id -); - - -create table observation_fact_&&upload_id as -select * from NightHeronData.observation_fact where 1=0; - -insert /*+ append */ into observation_fact_&&upload_id ( - patient_num, encounter_num, sub_encounter, - concept_cd, - provider_id, - start_date, - modifier_cd, - instance_num, - valtype_cd, - tval_char, - nval_num, - valueflag_cd, - units_cd, - end_date, - location_cd, - update_date, - import_date, upload_id, download_date, sourcesystem_cd) -select pmap.patient_num, - coalesce(emap.encounter_num, -abs(ora_hash(to_char(f.start_date, 'YYYYMMDD') || f.patient_ide))), -- TODO: pat_day func - f.encounter_ide, - f.concept_cd, - f.provider_id, - f.start_date, - f.modifier_cd, - f.instance_num, - f.valtype_cd, - f.tval_char, - f.nval_num, - f.valueflag_cd, - f.units_cd, - f.end_date, - f.location_cd, - f.update_date, - sysdate, up.upload_id, :download_date, up.source_cd -from &&epic_fact_view f - join NightHeronData.patient_mapping pmap - on pmap.patient_ide = f.patient_ide - and pmap.patient_ide_source = :pat_source_cd - and pmap.patient_ide_status = 'A' -- TODO: (select active from i2b2_status) - left join NightHeronData.encounter_mapping emap - on emap.encounter_ide = f.encounter_ide - and emap.encounter_ide_source = :enc_source_cd - and emap.encounter_ide_status = 'A' - , NightHeronData.upload_status up -where up.upload_id = :upload_id - and f.patient_ide between :pat_id_lo and :pat_id_hi - &&log_fact_exceptions -; -commit -; - - -/* For this upload of data, check primary key constraints. -This could perhaps be factored out of all the load scripts into i2b2_facts_deid.sql, -at the cost of slowing down the select count(*) for summary stats, below. - -TODO: check key constraint - -alter table observation_fact_&&upload_id - enable constraint observation_fact_pk - -- TODO: log errors ... ? #2117 - ; - */ - - -select count(*) -from NightHeronData.upload_status -where transform_name = :task_id and load_status = 'OK'; - diff --git a/Oracle/epic_flowsheets_transform.sql b/Oracle/epic_flowsheets_transform.sql deleted file mode 100644 index fc40305..0000000 --- a/Oracle/epic_flowsheets_transform.sql +++ /dev/null @@ -1,577 +0,0 @@ -/** epic_flowsheets_transform.sql -- prepare to load Epic EMR flowsheet facts - -Copyright (c) 2012 University of Kansas Medical Center -part of the HERON* open source codebase; see NOTICE file for license details. -* http://informatics.kumc.edu/work/wiki/HERON - - * See also epic_flowsheets_multiselect.sql, epic_etl.py, epic_facts_load.sql - * - * References: - * - * Lesson 4: Flowsheet Data - * Clarity Data Model - EpicCare Inpatient Spring 2008 Training Companion - * https://userweb.epic.com/epiclib/epicdoc/EpicWiseSpr08/Clarity/Clarity%20Training%20Companions/CLR209%20Data%20Model%20-%20EpicCare%20Inpatient/04TC%20Flowsheet%20Data.doc - */ - -select test_name from etl_tests where 'dep' = 'etl_tests_init.sql'; - --- Check that this is the ID instance -select flo_meas_id from CLARITY.ip_flwsht_meas where 1=0; - --- to see the time detail... -alter session set NLS_DATE_FORMAT = 'YYYY-MM-DD HH:MI'; - --- define heron_obs_sample = sample (0.001, 1234) - -create or replace view etl_test_domain_flowsheets as -select 'Flowsheets' test_domain from dual; - -insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) -with test_key as ( - select test_domain, - 'val_type_seen_before' test_name from etl_test_domain_flowsheets - ) -, test_values as ( - select count(*) test_value, test_key.* from ( - select distinct zcvt.name - from CLARITY.ip_flwsht_meas ifm - join CLARITY.ip_flo_gp_data ifgd - on ifm.flo_meas_id= ifgd.flo_meas_id - join CLARITY.zc_val_type zcvt on ifgd.val_type_c = zcvt.val_type_c - where zcvt.name not in ( - 'Blood Pressure', 'Category Type', 'Concentration', 'Custom List', - 'Date', 'Dose', 'Height', 'Numeric Type', 'Patient Height', - 'Patient Weight', 'Rate', 'String Type', 'Temperature', 'Time', 'Weight') - ), test_key - ) -select test_value, test_domain, test_name, sq_result_id.nextval, sysdate -from test_values -; - - -/*************** - * flo_meas_type -- per-flo_meas_id transformation - -Starting with relevant columns from ip_flo_gp_data, -determine i2b2 valtype_cd, UNITS_CD. - -See also i2b2_facts_deid.sql regarding identified/de-identified data -flag in valtype_cd. - -Values for height, weight, and temperature are (for reasons lost to history) -converted to SI units. - -Also, trivially set up MODIFIER_CD, CONFIDENCE_NUM. - -TODO: consider migrating to CSV spreadsheet. Too bad SQL doesn't - have relation constants a la R read.table() or JSON. - - */ --- compute i2b2 valtype_cd for each flo_meas_id -create or replace view flo_meas_type as -select ifgd.flo_meas_id, ifgd.disp_name - , ifgd.multi_sel_yn - , ifgd.record_state_c - , zcvt.name val_type - , case - when zcvt.name in ( - 'Numeric Type', - 'Blood Pressure', 'Temperature', 'Time', - 'Patient Weight', 'Weight', - 'Patient Height', 'Height') then 'N' - when zcvt.name in ('Custom List', 'Category Type') then '@' - -- only add ID information to nightheron - when zcvt.name in ('String Type') then 'Ti' - when zcvt.name in ('Date') then 'D' - else null -- val_type_seen_before test keeps these bounded - end valtype_cd - , 'KUH|FLO_MEAS_ID:' || ifgd.flo_meas_id CONCEPT_CD - - -- Convert to SI units per 2011 design. - , case - when zcvt.name in ( - /* Epic documentation doesn't give units for 'Weight', - so we've determined emperically that it's oz. - See also mean Birth Weight test based on flo_stats_num - in epic_flowsheets_multiselect.sql */ - 'Patient Weight', 'Weight') then 1.0 / 16.0 / 2.2 -- 16 oz/lb; 2.2kg/lb - when zcvt.name in ( - 'Patient Height', 'Height') then 2.54 -- 2.54 cm/in - when zcvt.name in ( - 'Temperature') then 5/9 -- F to C - else 1 - end scale - , case - when zcvt.name in ( - 'Temperature') then -32 -- F to C - else 0 - end bias - , case - when zcvt.name in ( - 'Patient Weight', 'Weight') then 'kg' - when zcvt.name in ( - 'Temperature') then 'C' - when zcvt.name in ( - 'Time') then 's' - when zcvt.name in ( - 'Patient Height', 'Height') then 'cm' - when zcvt.name in ( - 'Blood Pressure') then 'mmHg' - else ifgd.units - end UNITS_CD - , '@' MODIFIER_CD - , to_number(null) CONFIDENCE_NUM -from CLARITY.ip_flo_gp_data ifgd -left join CLARITY.zc_val_type zcvt on ifgd.val_type_c = zcvt.val_type_c -; - - -/*************** - * flowsheet_day -- per-patient-day transformation - -Starting with ip_flwsht_rec, compute ENCOUNTER_IDE, PATIENT_IDE -(to be mixed with patient_mapping and encounter_mapping in epic_facts_load.sql). - -Trivially set LOCATION_CD. - - */ -create or replace view flowsheet_day as -select ifr.record_date - , ifr.pat_id - , ifr.fsd_id - , ifr.INPATIENT_DATA_ID - -- i2b2 equivalents common to many/all datatypes - - -- patient day; see patient_day_visit view - , TO_CHAR(ifr.record_date,'YYYYMMDD') || ifr.pat_id - ENCOUNTER_IDE - , ifr.pat_id PATIENT_IDE - , '@' LOCATION_CD -from CLARITY.ip_flwsht_rec ifr -; - - -/*************** - * flowsheet_obs -- per-measurement transformation common to all datatypes - -Starting with ip_flwsht_meas, compute instance_num, -start_date, end_date, update_date. - -Filter out rows where meas_value is null. - -Trivially (for now) set PROVIDER_ID, valueflag_cd. - - */ -create or replace view flowsheet_obs as -select /*+ parallel(16) */ -- ISSUE: parameterize parallel_degree? - ifm.fsd_id - , ifm.line - , ifm.flo_meas_id - , ifm.meas_value - , ifm.recorded_time - , ifm.entry_time - -- per-measurement conversion to i2b2 terms - , '@' PROVIDER_ID -- todo: use ifm.TAKEN_USER_ID - , recorded_time START_DATE - , ifm.fsd_id * 100000 + ifm.line instance_num - , '@' valueflag_cd -- TODO #3850: [H]igh/[L]ow/[A]bnormal based on ifm.ABNORMAL_C - , recorded_time END_DATE - , entry_time UPDATE_DATE -from CLARITY.ip_flwsht_meas ifm -where meas_value is not null -; - - -/******************** - * flowsheet_num_data - handle number syntax - -Starting with flowsheet_obs, split diastolic and systolic into separate rows, -flag (but don't filter) illegal numerals, and convert the rest to_number(). - -Values such as 2.22222222222222E+22 don't fit in nval_num, -which is declared NUMBER(18,5). Let's throw out values using E notation. -The `numerals_filtered` test (in epic_flowsheets_multiselect) verifies that -we're not throwing away more than a handful. - -Stats in epic_flowsheets_multiselect.sql are based on this view, -before we join with per-patient-day or per-flow-measure info. - - */ -create or replace view flowsheet_num_data as -with -parts as ( - select flo_meas_id, bp_part, pattern - from flo_meas_type fmt - left join ( - select 'Blood Pressure' val_type, '_DIASTOLIC' bp_part, '\d+$' pattern from dual -union all select 'Blood Pressure' val_type, '_SYSTOLIC' bp_part, '^\d+' pattern from dual - ) parts on parts.val_type = fmt.val_type - where fmt.valtype_cd like 'N%' -) -, numerals as ( - select fsd_id, line - , obs.flo_meas_id - , bp_part - , meas_value - , recorded_time - , entry_time - , case - when parts.pattern is not null - then regexp_substr(meas_value, parts.pattern) - else meas_value - end numeral - , START_DATE - , END_DATE - , instance_num - , UPDATE_DATE - , PROVIDER_ID - , valueflag_cd - from flowsheet_obs obs - join parts on parts.flo_meas_id = obs.flo_meas_id -) -, num_check as ( - select numerals.* - , case - when numeral is null then null - when regexp_like(numeral, '^-?\d*(\.\d+)?$') then 1 - else 0 - end ok - from numerals -) - select num_check.* - , case - when ok = 1 then to_number(numeral) - else null - end meas_value_num - from num_check -; - - -/************** - * numerictypeflows - Numeric, Blood Pressure, Weight, etc. - -Build concept_cd using blood pressure suffix; apply units bias and scale; -trivially set TVAL_CHAR. - -Join flowsheet_num_data with flo_meas_type and flowsheet_day to fill in -remaining observation_fact style fields. - - */ -create or replace view numerictypeflows as -select ENCOUNTER_IDE - , PATIENT_IDE - , 'KUH|FLO_MEAS_ID:' || num_data.flo_meas_id || bp_part concept_cd - , PROVIDER_ID - , START_DATE - , MODIFIER_CD - , instance_num - , valtype_cd - , 'E' TVAL_CHAR -- i2b2 doc says 'EQ' but demo data says 'E' - , case when fmt.bias <> 0 or fmt.scale <> 1 - then - -- Round so as not to imply more precision than was measured. - round((meas_value_num + fmt.bias) * fmt.scale, 3) - else - meas_value_num - end nval_num - , valueflag_cd - , UNITS_CD - , END_DATE - , LOCATION_CD - , CONFIDENCE_NUM - , UPDATE_DATE -from flowsheet_num_data num_data -join flo_meas_type fmt on fmt.flo_meas_id = num_data.flo_meas_id -join flowsheet_day fsd on fsd.fsd_id = num_data.fsd_id -where num_data.ok = 1 -; - --- select * from numerictypeflows; - - -/*********** - * datemeasureflows - for value_type_name "Date" - * - * STRANGE! Syntax is EITHER: - * - a number of days since Dec 31, 1840 or - * - a date in 'MM/DD/YYYY' format - */ -create or replace view datemeasureflows as -Select - ENCOUNTER_IDE - , PATIENT_IDE - , CONCEPT_CD - , PROVIDER_ID - , START_DATE - , MODIFIER_CD - , instance_num - , 'D' VALTYPE_CD, - case - when substr(meas_value , 1, instr(meas_value , '/') -1 ) is NULL - then to_char(to_date('12/31/1840', 'MM/DD/YYYY') + meas_value, 'YYYY-MM-DD') - else to_char(to_date(meas_value, 'MM/DD/YYYY'), 'YYYY-MM-DD') - end TVAL_CHAR - , null NVAL_NUM - , valueflag_cd - , UNITS_CD - , END_DATE - , LOCATION_CD - , CONFIDENCE_NUM - , UPDATE_DATE -from flowsheet_obs obs -join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id -join flowsheet_day fsd on fsd.fsd_id = obs.fsd_id -where valtype_cd like 'D%'; - - -/* todo:: test date-shifting when VALUE_TYPE_NAME='Date' - e.g. 11568 or 700 PLACEMENT DATE (#299) */ - - -/***************** - * idstringtypeflows -- free text - -For measures identifed (in flo_meas_type) as free text, set tval_char. - -Trivially set nval_num. - */ -create or replace view idstringtypeflows as -select - fsd.ENCOUNTER_IDE - , fsd.PATIENT_IDE - , fmt.CONCEPT_CD - , obs.PROVIDER_ID - , obs.START_DATE - , fmt.MODIFIER_CD - , obs.instance_num - , fmt.VALTYPE_CD - , obs.meas_value TVAL_CHAR - , to_number(null) NVAL_NUM - , obs.valueflag_cd - , fmt.UNITS_CD - , obs.END_DATE - , fsd.LOCATION_CD - , fmt.CONFIDENCE_NUM - , obs.UPDATE_DATE -from flowsheet_obs obs -join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id -join flowsheet_day fsd on fsd.fsd_id = obs.fsd_id -where valtype_cd like 'T%'; - - -/***************** - * deidstringtypeflows -- deid-ed free text - -For measures identifed (in flo_meas_type) as free text, and have corresponding -de-identified meas_values. - -As an intermediate step create `approved_deid_flowsheets` containing all of the -de-identified free text notes that have been approved for inclusion. - -Trivially set nval_num. - */ - - -whenever sqlerror continue; -drop table approved_deid_flowsheets; -whenever sqlerror exit; - - -create table approved_deid_flowsheets ( - fsd_id number - , line number - , flo_meas_id number - , meas_value varchar(4000) - , primary key (fsd_id, line)); - - -whenever sqlerror continue; -drop table approved_flo_meas_ids; --- Oracle gave really bad performance while `deidentify.deid_flowsheet_approval` --- was a view. -create table approved_flo_meas_ids as -select * from deidentify.deid_flowsheet_approval where approved_to_deid = 1; - -insert into approved_deid_flowsheets -(fsd_id, line, flo_meas_id, meas_value) -select fsd_id, line, flo_meas_id, meas_value -from ( - -- `fsd_id` and `line` are extracted from `flo_notes.id` - -- see: heron_staging/deidentify/Flowsheets-DEID-Stage.sql - select to_number(substr(to_char(id), - 1, - length(to_char(id))-5)) as fsd_id - , to_number(substr(id, -5)) as line - , to_number(note_id) as flo_meas_id - , deid_note_text as meas_value - from deidentify.flo_notes) deid - -- Include only meas_values that have flo_meas_id approved. -where exists (select * from approved_flo_meas_ids af - where deid.flo_meas_id = af.flo_meas_id - ); -whenever sqlerror exit; - - -insert into etl_test_values (test_value, test_domain, test_name, result_id, - result_date) -with test_key as ( - select test_domain, - 'staged_deid_flowsheet' test_name from etl_test_domain_flowsheets - ) -, test_values as ( - select count(*) test_value, test_key.* from approved_deid_flowsheets, test_key - ) -select test_value, test_domain, test_name, sq_result_id.nextval, sysdate -from test_values -; - --- Did `staged_deid_flowsheet` fail? First check to see if the necessary --- staged tables exist. - --- select * from deidentify.flo_notes; --- select * from deidentify.deid_flowsheet_approval; - -create or replace view deidstringtypeflows as -select - fsd.ENCOUNTER_IDE - , fsd.PATIENT_IDE - , fmt.CONCEPT_CD - , obs.PROVIDER_ID - , obs.START_DATE - , fmt.MODIFIER_CD - , obs.instance_num - , 'Td' as valtype_cd - , deid.meas_value TVAL_CHAR - , to_number(null) NVAL_NUM - , obs.valueflag_cd - , fmt.UNITS_CD - , obs.END_DATE - , fsd.LOCATION_CD - , fmt.CONFIDENCE_NUM - , obs.UPDATE_DATE -from flowsheet_obs obs -join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id -join flowsheet_day fsd on fsd.fsd_id = obs.fsd_id -join approved_deid_flowsheets deid - on deid.fsd_id = obs.fsd_id - and deid.line = obs.line -where valtype_cd like 'T%'; - - -/******************** - * flowsheet_nom_data - nominal data, including multi select - -Stats in epic_flowsheets_multiselect.sql are based on this view, -before we join with per-patient-day or per-flow-measure info. - -The flow_measure_multi table (eventually) maps 'x;y;z' multi-select values -to the constituent 'x', 'y', and 'z' values. Create the table so we can -refer to it, but leave populating it to epic_flowsheets_multiselect.sql. - - */ -whenever sqlerror continue; -drop table flow_measure_multi; -whenever sqlerror exit; -create table flow_measure_multi as -select fm.meas_value - , 0 val_ix - , fm.meas_value choice_label -from flowsheet_obs fm -where 1=0; - -create or replace view flowsheet_nom_data as - select fsd_id, obs.line - , obs.flo_meas_id - , fmt.multi_sel_yn - , obs.meas_value - , fmm.val_ix - , fmm.choice_label multi_choice_label - , coalesce(fmm.choice_label, obs.meas_value) choice_label - , recorded_time - , entry_time - , START_DATE - , END_DATE - , UPDATE_DATE - , PROVIDER_ID - , valueflag_cd - from flowsheet_obs obs - join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id - and fmt.valtype_cd = '@' -- valtype_cd for nominal data - left join flow_measure_multi fmm - -- Try not to look in flow_measure_multi unless - -- except when we have multi_sel_yn and a ';' . - on fmm.meas_value = case - when fmt.multi_sel_yn = '1' - and obs.meas_value like '%;%' - then obs.meas_value - else null - end -; - --- "forward reference" flo_stats_nom table for use in flowsheets_concepts.sql -whenever sqlerror continue; -drop table flo_stats_nom; -whenever sqlerror exit; - -create table flo_stats_nom as -select flo_meas_id, meas_value choice_label - , 0 qty - , 0 tot - , 99.9 pct - , date '2001-01-01' recorded_time_min - , date '2001-01-01' recorded_time_max -from flowsheet_nom_data -where 1 = 0 -; - - -/************* - * selectflows -- Custom List nominal observations - -Compute concept_cd using flo_meas_id and ip_flo_cust_list.line where available; -fall back to hash of choice label. - -Trivially set tval_char, nval_num. - */ - -create or replace view selectflows as -Select ENCOUNTER_IDE - , PATIENT_IDE - , (case when ifcl.line is not null - then 'KUH|FLO_MEAS_ID+LINE:' || obs.flo_meas_id || '_' || ifcl.line - else 'KUH|FLO_MEAS_ID+hash:' || obs.flo_meas_id || '_' || ora_hash(obs.choice_label) - end) CONCEPT_CD - , PROVIDER_ID - , START_DATE - , MODIFIER_CD - -- Handle cases such as 'Sinus Tachycardia;ST' - -- where multiple choices map to the same ifcl.line - -- by including the val_ix (1, 2) in instance_num. - , (obs.fsd_id * 100000 + obs.line) * 10 + coalesce(val_ix, 0) instance_num - , VALTYPE_CD - , '@' TVAL_CHAR - , to_number(null) NVAL_NUM - , valueflag_cd - , UNITS_CD - , END_DATE - , LOCATION_CD - , CONFIDENCE_NUM - , UPDATE_DATE -from flowsheet_nom_data obs -join flo_meas_type fmt on fmt.flo_meas_id = obs.flo_meas_id -join flowsheet_day fsd on fsd.fsd_id = obs.fsd_id -left join CLARITY.ip_flo_cust_list ifcl - on ifcl.flo_meas_id = obs.flo_meas_id - and (obs.choice_label = ifcl.abbreviation or - obs.choice_label = ifcl.custom_value) -where valtype_cd = '@'; - --- select * from multiselectflows where rownum < 200; - - -/* complete check */ -create or replace view epic_flowsheets_txform_sql as -select &&design_digest design_digest from dual; - -select 1 up_to_date -from epic_flowsheets_txform_sql where design_digest = &&design_digest; diff --git a/epic_flowsheets.py b/epic_flowsheets.py deleted file mode 100644 index 1466131..0000000 --- a/epic_flowsheets.py +++ /dev/null @@ -1,192 +0,0 @@ -'''epic_flowsheets -- ETL tasks for Epic Flowsheets to i2b2 -''' - -from typing import List - -import luigi -import sqlalchemy as sqla - -from etl_tasks import ( - DBAccessTask, I2B2ProjectCreate, I2B2Task, SourceTask, - SqlScriptTask, UploadTask, - DBTarget, SchemaTarget, UploadTarget -) -from script_lib import Script -from sql_syntax import Environment, Params -import param_val as pv - - -class CLARITYExtract(SourceTask, DBAccessTask): - download_date = pv.TimeStampParam(description='see client.cfg') - source_cd = pv.StrParam(default="Epic@kumed.com") - - # ISSUE: parameterize CLARITY schema name? - schema = 'CLARITY' - table_eg = 'patient' - - def _dbtarget(self) -> DBTarget: - return SchemaTarget(self._make_url(self.account), - schema_name=self.schema, - table_eg=self.table_eg, - echo=self.echo) - - -class NightHeronCreate(I2B2ProjectCreate): - pass - - -class NightHeronTask(I2B2Task): - '''Mix in identified star_schema parameter config. - ''' - @property - def project(self) -> I2B2ProjectCreate: - return NightHeronCreate() - - -class FromEpic(NightHeronTask): - '''Mix in source and substitution variables for Epic ETL scripts. - ''' - @property - def source(self) -> CLARITYExtract: - return CLARITYExtract() - - @property - def variables(self) -> Environment: - return self.vars_for_deps - - @property - def vars_for_deps(self) -> Environment: - # TODO: config = [] - # TODO: design = [] # TODO - return dict() - - -class FlowsheetViews(SqlScriptTask): - # TODO: subsume this in a load task - # TODO: patient survey using ntile() -- persistent worthwhile? - script = Script.epic_flowsheets_transform - - -class EpicDimensionsLoad(FromEpic, DBAccessTask): - '''Placeholder for heron_load/load_epic_dimensions.sql - ''' - @property - def transform_name(self) -> str: - return 'load_epic_dimensions' - - def complete(self) -> bool: - return self.output().exists() - - def output(self) -> luigi.Target: - return self._upload_target() - - def _upload_target(self) -> 'UploadTarget': - return UploadTarget(self._make_url(self.account), - self.project.upload_table, - self.transform_name, self.source, - echo=self.echo) - - def run(self) -> None: - raise NotImplementedError - - -class PatIdMapping(luigi.WrapperTask): - def requires(self) -> List[luigi.Task]: - # TODO: split pat_id mapping out of load_epic_dimensions.sql - return [EpicDimensionsLoad()] - - -class FSDMapping(luigi.WrapperTask): - def requires(self) -> List[luigi.Task]: - # TODO: split pat_id mapping out of load_epic_dimensions.sql - return [EpicDimensionsLoad()] - - -class EpicFactsLoadGroup(FromEpic, UploadTask): - epic_fact_view = pv.StrParam() - pat_id_lo = pv.StrParam() - pat_id_hi = pv.StrParam() - pat_group_num = pv.IntParam(significant=False) - pat_group_qty = pv.IntParam(significant=False) - pat_source_cd = pv.StrParam(default='Epic@kumed.com') - enc_source_cd = pv.StrParam() - - script = Script.epic_facts_load - - @property - def label(self) -> str: - return '{view} #{group}: {lo} to {hi}'.format( - view=self.epic_fact_view, group=self.pat_group_num, - lo=self.pat_id_lo, hi=self.pat_id_hi) - - def requires(self) -> List[luigi.Task]: - return UploadTask.requires(self) + [ - self.source, - FSDMapping(), - PatIdMapping(), - ] - - @property - def variables(self) -> Environment: - return dict(self.vars_for_deps, - epic_fact_view=self.epic_fact_view, - log_fact_exceptions='') - - def script_params(self) -> Params: - return dict(UploadTask.script_params(self), - pat_source_cd=self.pat_source_cd, - enc_source_cd=self.enc_source_cd, - pat_id_lo=self.pat_id_lo, - pat_id_hi=self.pat_id_hi) - - -class FlowsheetsLoad(FromEpic, DBAccessTask, luigi.WrapperTask): - pat_group_qty = pv.IntParam(default=5, significant=False) - enc_source_cd = pv.StrParam(default='Epic+pat_id_day@kumed.com') - - # issue: parameterize fact views? - fact_views = [ - 'numerictypeflows', - 'numerictypeflows', - 'datemeasureflows', - 'selectflows', - 'idstringtypeflows', - 'deidstringtypeflows', - ] - - pat_grp_q = ''' - select :group_qty grp_qty, group_num - , min(pat_id) pat_id_lo - , max(pat_id) pat_id_hi - from ( - select pat_id - , ntile(:group_qty) over (order by pat_id) as group_num - from ( - select /*+ parallel(20) */ distinct pat_id - from clarity.patient - where pat_id is not null -- help Oracle use the index - ) ea - ) w_ntile - group by group_num, :group_qty - order by group_num - ''' - - def requires(self) -> List[luigi.Task]: - groups = self.partition_patients() - return [ - EpicFactsLoadGroup(epic_fact_view=view, - enc_source_cd=self.enc_source_cd, - pat_id_lo=lo, - pat_id_hi=hi, - pat_group_qty=qty, - pat_group_num=num) - for view in self.fact_views - for (qty, num, lo, hi) in groups] - - def partition_patients(self) -> List[sqla.engine.RowProxy]: - # ISSUE: persist partition? - with self.connection('partition patients') as q: - groups = q.execute(self.pat_grp_q.format(i2b2_star=self.project.star_schema), - params=dict(group_qty=self.pat_group_qty)).fetchall() - q.log.info('groups: %s', groups) - return groups diff --git a/script_lib.py b/script_lib.py index 053d21c..1e6f8f4 100644 --- a/script_lib.py +++ b/script_lib.py @@ -5,13 +5,13 @@ Each script should have a title, taken from the first line:: >>> Script.med_admin.title - 'Build Medication Administration table.' + 'create and populate the med_admin table.' >>> text = Script.med_admin.value >>> lines = text.split('\n') >>> print(lines[0]) ... #doctest: +NORMALIZE_WHITESPACE - /** med_admin - Build Medication Administration table. + /** med_admin - create and populate the med_admin table. We can separate the script into statements:: @@ -32,46 +32,17 @@ >>> Script.sqlerror('select 1 + 1 from dual') is None True -Dependencies between scripts are declared as follows:: - - >>> print(next(decl for decl in statements if "'dep'" in decl)) - ... #doctest: +ELLIPSIS - select test_name from etl_tests where 'dep' = 'etl_tests_init.sql' - - >>> Script.epic_flowsheets_transform.deps() - ... #doctest: +ELLIPSIS - [] - -We statically detect relevant effects; i.e. tables and views created:: - - >>> Script.epic_flowsheets_transform.created_objects() - ... #doctest: +ELLIPSIS - [view etl_test_domain_flowsheets, view flo_meas_type, view flowsheet_day, ... - -as well as tables inserted into:: - - >>> variables={I2B2STAR: 'I2B2DEMODATA', - ... CMS_RIF: 'CMS_DEID', 'upload_id': '20', 'chunk_qty': 20, - ... 'cms_source_cd': "'ccwdata.org'", 'source_table': 'T'} - >>> Script.epic_flowsheets_transform.inserted_tables(variables) - ['etl_test_values', 'approved_deid_flowsheets', 'etl_test_values'] - The last statement should be a scalar query that returns non-zero to signal that the script is complete: >>> print(statements[-1]) ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE - select 1 up_to_date - from epic_flowsheets_txform_sql where design_digest = &&design_digest + select 1 from cdm_status where status = 'med_admin' The completion test may depend on a digest of the script and its dependencies: - >>> design_digest = Script.epic_flowsheets_transform.digest() - >>> last = Script.epic_flowsheets_transform.statements(variables)[-1].strip() - >>> print(last) - ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE - select 1 up_to_date - from epic_flowsheets_txform_sql where design_digest = ... + >>> Script.med_admin.digest() != 0 + True ISSUE : Python hashes are senstive to the machine running the test? @@ -80,8 +51,8 @@ partitions; these scripts must not refer to such variables in their completion query: - >>> del variables['upload_id'] - >>> print(Script.migrate_fact_upload.statements(variables, + ..> del variables['upload_id'] + ..> print(Script.migrate_fact_upload.statements(variables, ... skip_unbound=True)[-1].strip()) commit @@ -188,12 +159,12 @@ def digest(self) -> int: def _text(self) -> List[str]: '''Get the text of this script and its dependencies. - >>> nodeps = Script.migrate_fact_upload + >>> nodeps = Script.med_admin >>> nodeps._text() == [nodeps.value] True - >>> complex = Script.epic_flowsheets_transform - >>> complex._text() != [complex.value] + ..> complex = Script.epic_flowsheets_transform + ..> complex._text() != [complex.value] True ''' return sorted(set(s.sql for s in self.dep_closure())) diff --git a/sql_syntax.py b/sql_syntax.py index e44d11c..c5e5a6b 100644 --- a/sql_syntax.py +++ b/sql_syntax.py @@ -202,9 +202,9 @@ def created_objects(statement: SQL) -> List[ObjectId]: >>> created_objects('create or replace view x\nas ...') [view x] ''' - m = re.search('^create or replace view (\S+)', statement.strip()) + m = re.search('(?i)^create or replace view (\S+)', statement.strip()) views = [ViewId(m.group(1))] if m else [] # type: List[ObjectId] - m = re.search('^create table (\S+)', statement.strip()) + m = re.search('(?i)^create table (\w+)', statement.strip()) tables = [TableId(m.group(1))] if m else [] # type: List[ObjectId] return tables + views From fe01cd68e43cebc1992691a72b637b1fcd432dc1 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 7 May 2018 15:23:52 -0500 Subject: [PATCH 367/507] code style fixes: whitespace etc. per CONTRIBUTING.md and setup.cfg and PEP8 --- Oracle/backup_cdm.py | 1 + csv_load.py | 6 +++--- etl_tasks.py | 13 ++++++++----- i2p_tasks.py | 2 ++ setup.cfg | 2 +- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Oracle/backup_cdm.py b/Oracle/backup_cdm.py index 84c8266..2d54e55 100644 --- a/Oracle/backup_cdm.py +++ b/Oracle/backup_cdm.py @@ -86,6 +86,7 @@ def fetchall(self): eprint('fetch returning: ' + str(r)) return r + if __name__ == '__main__': def _tcb(): from os import environ diff --git a/csv_load.py b/csv_load.py index 6382c10..a753354 100644 --- a/csv_load.py +++ b/csv_load.py @@ -11,13 +11,13 @@ log = logging.getLogger(__name__) + class LoadCSV(DBAccessTask): tablename = StrParam() csvname = StrParam() rowcount = IntParam(default=1) def complete(self) -> bool: - #return False db = self._dbtarget().engine table = Table(self.tablename, sqla.MetaData()) if not table.exists(bind=db): @@ -25,7 +25,7 @@ def complete(self) -> bool: return False with self.connection() as q: actual = q.scalar('select records from cdm_status where status = \'%s\'' % self.tablename) - actual = 0 if actual == None else actual + actual = 0 if actual is None else actual log.info('table %s has %d rows', self.tablename, actual) return actual >= self.rowcount # type: ignore # sqla @@ -66,4 +66,4 @@ def setStatus(self) -> None: with self.connection() as q: actual = q.scalar(sqla.select([func.count()]).select_from(self.tablename)) - db.execute(statusTable.insert(), [{'STATUS':self.tablename, 'LAST_UPDATE':datetime.now(), 'RECORDS':actual}]) \ No newline at end of file + db.execute(statusTable.insert(), [{'STATUS': self.tablename, 'LAST_UPDATE': datetime.now(), 'RECORDS': actual}]) diff --git a/etl_tasks.py b/etl_tasks.py index 14948ca..1e21634 100644 --- a/etl_tasks.py +++ b/etl_tasks.py @@ -233,11 +233,14 @@ def vars_for_deps(self) -> Environment: def requires(self) -> List[luigi.Task]: '''Wrap each of `self.script.deps()` in a SqlScriptTask. ''' - return [type(s.name, (SqlScriptTask,), {'script':s, - 'param_vars':self.vars_for_deps, - 'account':self.account, - 'passkey':self.passkey, - 'echo':self.echo})() + return [type(s.name, (SqlScriptTask,), + { + 'script': s, + 'param_vars': self.vars_for_deps, + 'account': self.account, + 'passkey': self.passkey, + 'echo': self.echo + })() for s in self.script.deps()] def log_info(self) -> Dict[str, Any]: diff --git a/i2p_tasks.py b/i2p_tasks.py index 5069c00..54a88e4 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -10,6 +10,7 @@ from sqlalchemy.exc import DatabaseError from typing import List + class CDMScriptTask(SqlScriptTask): @property @@ -84,6 +85,7 @@ def requires(self): lab_result_cm(), med_admin(), obs_clin(), obs_gen(), pcornet_trial(), prescribing(), pro_cm(), procedures(), provider(), vital()] + class lab_result_cm(CDMScriptTask): script = Script.lab_result_cm diff --git a/setup.cfg b/setup.cfg index 3c3a387..442184a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,7 @@ ignore = E731, E126 # guide, for window sizing: # 23456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 max-line-length = 120 -exclude = pythonjsonlogger +exclude = pythonjsonlogger,ADD_SCILHS_100 [mypy] From 5ef3d85ca89d128fd2395bc08e853e245938d6eb Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 7 May 2018 15:33:08 -0500 Subject: [PATCH 368/507] type hints work again; organize imports Organize imports by stdlib, 3rd party, 1st party per PEP8. --- csv_load.py | 14 +++++++++----- eventlog.py | 3 ++- i2p_tasks.py | 50 +++++++++++++++++++++++++++----------------------- 3 files changed, 38 insertions(+), 29 deletions(-) diff --git a/csv_load.py b/csv_load.py index a753354..05220dc 100644 --- a/csv_load.py +++ b/csv_load.py @@ -1,10 +1,13 @@ from collections import defaultdict from csv import DictReader from datetime import datetime +from typing import Dict + +from sqlalchemy import func, MetaData, Table, Column # type: ignore +from sqlalchemy.types import String # type: ignore + from etl_tasks import DBAccessTask from param_val import StrParam, IntParam -from sqlalchemy import func, MetaData, Table, Column -from sqlalchemy.types import String import logging import sqlalchemy as sqla @@ -34,17 +37,18 @@ def run(self) -> None: self.setStatus() def load(self) -> None: - def sz(l, chunk=16): + def sz(l: int, chunk: int=16) -> int: return max(chunk, chunk * ((l + chunk - 1) // chunk)) db = self._dbtarget().engine schema = MetaData() l = list() - with open(self.csvname) as fin: + with open(self.csvname) as fin: # ISSUE: ambient dr = DictReader(fin) - mcl = defaultdict(int) + Dict # for tools that don't see type: comments. + mcl = defaultdict(int) # type: Dict[str, int] for row in dr: l.append(row) for col in dr.fieldnames: diff --git a/eventlog.py b/eventlog.py index 1de5799..30a6c12 100644 --- a/eventlog.py +++ b/eventlog.py @@ -74,7 +74,8 @@ def __init__(self, logger: logging.Logger, event: JSONObject, List # let flake8 know we're using it def __repr__(self) -> str: - return '%s(%s, %s)' % (self.__class__.__name__, self.name, self.event) + # typeshed doesn't know that LoggerAdapter has .name? + return '%s(%s, %s)' % (self.__class__.__name__, self.name, self.event) # type: ignore def process(self, msg: str, kwargs: KWArgs) -> Tuple[str, KWArgs]: extra = dict(kwargs.get('extra', {}), diff --git a/i2p_tasks.py b/i2p_tasks.py index 54a88e4..dae3c27 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -1,14 +1,17 @@ """i2p_tasks -- Luigi CDM task support. """ +from typing import List + +import luigi +from sqlalchemy.engine import RowProxy +from sqlalchemy.exc import DatabaseError + from csv_load import LoadCSV from etl_tasks import SqlScriptTask from param_val import IntParam from script_lib import Script -from sql_syntax import Environment -from sqlalchemy.engine import RowProxy -from sqlalchemy.exc import DatabaseError -from typing import List +from sql_syntax import Environment, Params class CDMScriptTask(SqlScriptTask): @@ -24,63 +27,63 @@ def variables(self) -> Environment: class condition(CDMScriptTask): script = Script.condition - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] class death(CDMScriptTask): script = Script.death - def requires(self): + def requires(self) -> List[luigi.Task]: return [demographic()] class death_cause(CDMScriptTask): script = Script.death_cause - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] class demographic(CDMScriptTask): script = Script.demographic - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] class diagnosis(CDMScriptTask): script = Script.diagnosis - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] class dispensing(CDMScriptTask): script = Script.dispensing - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] class encounter(CDMScriptTask): script = Script.encounter - def requires(self): + def requires(self) -> List[luigi.Task]: return [demographic()] class enrollment(CDMScriptTask): script = Script.enrollment - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] class harvest(CDMScriptTask): script = Script.harvest - def requires(self): + def requires(self) -> List[luigi.Task]: return [condition(), death(), death_cause(), diagnosis(), dispensing(), enrollment(), lab_result_cm(), med_admin(), obs_clin(), obs_gen(), pcornet_trial(), prescribing(), pro_cm(), procedures(), provider(), vital()] @@ -89,14 +92,14 @@ def requires(self): class lab_result_cm(CDMScriptTask): script = Script.lab_result_cm - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] class med_admin(CDMScriptTask): script = Script.med_admin - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] @@ -126,7 +129,8 @@ def results(self) -> List[RowProxy]: chunk_num <= :chunk_max) order by chunk_num ''' - params = dict(chunk_max=self.patient_chunk_max, chunk_qty=self.patient_chunks) + Params + params = dict(chunk_max=self.patient_chunk_max, chunk_qty=self.patient_chunks) # type: Params try: return lc.execute(q, params=params).fetchall() @@ -137,42 +141,42 @@ def results(self) -> List[RowProxy]: class pcornet_init(CDMScriptTask): script = Script.pcornet_init - def requires(self): + def requires(self) -> List[luigi.Task]: return [loadLabNormal(), loadHarvestLocal()] class pcornet_loader(CDMScriptTask): script = Script.pcornet_loader - def requires(self): + def requires(self) -> List[luigi.Task]: return [harvest()] class pcornet_trial(CDMScriptTask): script = Script.pcornet_trial - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] class prescribing(CDMScriptTask): script = Script.prescribing - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] class pro_cm(CDMScriptTask): script = Script.pro_cm - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] class procedures(CDMScriptTask): script = Script.procedures - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] @@ -183,7 +187,7 @@ class provider(CDMScriptTask): class vital(CDMScriptTask): script = Script.vital - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] From b1ef59caac6a927e0a3371474d65bba93ce5179b Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 10:43:36 -0500 Subject: [PATCH 369/507] NPI loader and end_time update for all tables. --- Oracle/condition.sql | 7 +- Oracle/death.sql | 7 +- Oracle/death_cause.sql | 7 +- Oracle/demographic.sql | 7 +- Oracle/diagnosis.sql | 7 +- Oracle/dispensing.sql | 7 +- Oracle/encounter.sql | 7 +- Oracle/enrollment.sql | 7 +- Oracle/harvest.sql | 7 +- Oracle/lab_result_cm.sql | 356 +++++++++++++++----------------------- Oracle/med_admin.sql | 7 +- Oracle/obs_clin.sql | 8 +- Oracle/obs_gen.sql | 8 +- Oracle/pcornet_init.sql | 16 +- Oracle/pcornet_loader.sql | 7 +- Oracle/pcornet_trial.sql | 7 +- Oracle/prescribing.sql | 8 +- Oracle/pro_cm.sql | 7 +- Oracle/procedures.sql | 7 +- Oracle/provider.sql | 36 +++- Oracle/vital.sql | 7 +- csv_load.py | 42 +---- etl_tasks.py | 30 +++- i2p_tasks.py | 89 +++++++++- 24 files changed, 410 insertions(+), 288 deletions(-) diff --git a/Oracle/condition.sql b/Oracle/condition.sql index eef156e..1fae8f9 100644 --- a/Oracle/condition.sql +++ b/Oracle/condition.sql @@ -1,5 +1,7 @@ /** condition - create and populate the condition table. */ +insert into cdm_status (task, start_time) select 'condition', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE condition'); END; @@ -102,6 +104,9 @@ BEGIN PCORNetCondition(); END; / -insert into cdm_status (status, last_update, records) select 'condition', sysdate, count(*) from condition +update cdm_status +set end_time = sysdate +set records = (select count(*) from condition) +where task = 'condition' / select 1 from cdm_status where status = 'condition' \ No newline at end of file diff --git a/Oracle/death.sql b/Oracle/death.sql index e30a70c..b0a61c8 100644 --- a/Oracle/death.sql +++ b/Oracle/death.sql @@ -1,5 +1,7 @@ /** death - create and populate the death table. */ +insert into cdm_status (task, start_time) select 'death', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE death'); END; @@ -45,7 +47,10 @@ BEGIN PCORNetDeath(); END; / -insert into cdm_status (status, last_update, records) select 'death', sysdate, count(*) from death +update cdm_status +set end_time = sysdate +set records = (select count(*) from death) +where task = 'death' / select 1 from cdm_status where status = 'death' diff --git a/Oracle/death_cause.sql b/Oracle/death_cause.sql index b6fda62..f0ac072 100644 --- a/Oracle/death_cause.sql +++ b/Oracle/death_cause.sql @@ -1,5 +1,7 @@ /** death_cause - create the death_cause table. */ +insert into cdm_status (task, start_time) select 'death_cause', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE death_cause'); END; @@ -13,7 +15,10 @@ CREATE TABLE death_cause( DEATH_CAUSE_CONFIDENCE varchar(2) NULL ) / -insert into cdm_status (status, last_update, records) select 'death_cause', sysdate, count(*) from death_cause +update cdm_status +set end_time = sysdate +set records = (select count(*) from death_cause) +where task = 'death_cause' / select 1 from cdm_status where status = 'death_cause' --SELECT count(PATID) from death_cause where rownum = 1 \ No newline at end of file diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql index e8daee1..1f44b3b 100644 --- a/Oracle/demographic.sql +++ b/Oracle/demographic.sql @@ -1,5 +1,7 @@ /** demographic - create and populate the demographic table. */ +insert into cdm_status (task, start_time) select 'demographic', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE demographic'); END; @@ -208,6 +210,9 @@ BEGIN PCORNetDemographic(); END; / -insert into cdm_status (status, last_update, records) select 'demographic', sysdate, count(*) from demographic +update cdm_status +set end_time = sysdate +set records = (select count(*) from demographic) +where task = 'demographic' / select 1 from cdm_status where status = 'demographic' \ No newline at end of file diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 514cfb4..2a8bb3f 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -1,5 +1,7 @@ /** diagnosis - create and populate the diagnosis table. */ +insert into cdm_status (task, start_time) select 'diagnosis', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE diagnosis'); END; @@ -245,7 +247,10 @@ BEGIN PCORNetDiagnosis(); END; / -insert into cdm_status (status, last_update, records) select 'diagnosis', sysdate, count(*) from diagnosis +update cdm_status +set end_time = sysdate +set records = (select count(*) from diagnosis) +where task = 'diagnosis' / select 1 from cdm_status where status = 'diagnosis' --SELECT count(DIAGNOSISID) from diagnosis where rownum = 1 \ No newline at end of file diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index 094ea7e..8f9b2cd 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -1,5 +1,7 @@ /** dispensing - create and populate the dispensing table. */ +insert into cdm_status (task, start_time) select 'dispensing', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE dispensing'); END; @@ -178,7 +180,10 @@ BEGIN PCORNetDispensing(); END; / -insert into cdm_status (status, last_update, records) select 'dispensing', sysdate, count(*) from dispensing +update cdm_status +set end_time = sysdate +set records = (select count(*) from dispensing) +where task = 'dispensing' / select 1 from cdm_status where status = 'dispensing' --SELECT count(DISPENSINGID) from dispensing where rownum = 1 \ No newline at end of file diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index a009063..bb2d551 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -1,5 +1,7 @@ /** encounter - create and populate the encounter table. */ +insert into cdm_status (task, start_time) select 'encounter', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE encounter'); END; @@ -99,7 +101,10 @@ BEGIN PCORNetEncounter(); END; / -insert into cdm_status (status, last_update, records) select 'encounter', sysdate, count(*) from encounter +update cdm_status +set end_time = sysdate +set records = (select count(*) from encounter) +where task = 'encounter' / select 1 from cdm_status where status = 'encounter' --SELECT count(ENCOUNTERID) from encounter where rownum = 1 \ No newline at end of file diff --git a/Oracle/enrollment.sql b/Oracle/enrollment.sql index b51c825..5d1282e 100644 --- a/Oracle/enrollment.sql +++ b/Oracle/enrollment.sql @@ -1,5 +1,7 @@ /** enrollment - create and populate the enrollment table. */ +insert into cdm_status (task, start_time) select 'enrollment', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE enrollment'); END; @@ -50,7 +52,10 @@ BEGIN PCORNetEnroll(); END; / -insert into cdm_status (status, last_update, records) select 'enrollment', sysdate, count(*) from enrollment +update cdm_status +set end_time = sysdate +set records = (select count(*) from enrollment) +where task = 'enrollment' / select 1 from cdm_status where status = 'enrollment' --SELECT count(PATID) from enrollment where rownum = 1 \ No newline at end of file diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index 359282e..f8f269a 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -1,5 +1,7 @@ /** harvest - create and populate the harvest table. */ +insert into cdm_status (task, start_time) select 'harvest', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE harvest'); END; @@ -101,6 +103,9 @@ BEGIN PCORNetHarvest(); END; / -insert into cdm_status (status, last_update, records) select 'harvest', sysdate, count(*) from harvest +update cdm_status +set end_time = sysdate +set records = (select count(*) from harvest) +where task = 'harvest' / select 1 from cdm_status where status = 'harvest' diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 534f40e..0b3e29a 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -1,243 +1,175 @@ /** lab_result_cm - create and populate the lab_result_cm table. */ +insert into cdm_status (task, start_time) select 'lab_result_cm', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE lab_result_cm'); END; / -CREATE TABLE lab_result_cm( - LAB_RESULT_CM_ID varchar(19) primary key, - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NULL, - LAB_NAME varchar(10) NULL, - SPECIMEN_SOURCE varchar(10) NULL, - LAB_LOINC varchar(10) NULL, - PRIORITY varchar(2) NULL, - RESULT_LOC varchar(2) NULL, - LAB_PX varchar(11) NULL, - LAB_PX_TYPE varchar(2) NULL, - LAB_ORDER_DATE date NULL, - SPECIMEN_DATE date NULL, - SPECIMEN_TIME varchar(5) NULL, - RESULT_DATE date NULL, - RESULT_TIME varchar(5) NULL, - RESULT_QUAL varchar(12) NULL, - RESULT_NUM number (15,8) NULL, - RESULT_MODIFIER varchar(2) NULL, - RESULT_UNIT varchar(11) NULL, - NORM_RANGE_LOW varchar(10) NULL, - NORM_MODIFIER_LOW varchar(2) NULL, - NORM_RANGE_HIGH varchar(10) NULL, - NORM_MODIFIER_HIGH varchar(2) NULL, - ABN_IND varchar(2) NULL, - RAW_LAB_NAME varchar(50) NULL, - RAW_LAB_CODE varchar(50) NULL, - RAW_PANEL varchar(50) NULL, - RAW_RESULT varchar(50) NULL, - RAW_UNIT varchar(50) NULL, - RAW_ORDER_DEPT varchar(50) NULL, - RAW_FACILITY_CODE varchar(50) NULL -) -/ BEGIN -PMN_DROPSQL('DROP SEQUENCE lab_result_cm_seq'); +PMN_DROPSQL('drop table lab_result_key'); END; / -create sequence lab_result_cm_seq -/ -create or replace trigger lab_result_cm_trg -before insert on lab_result_cm -for each row -begin - select lab_result_cm_seq.nextval into :new.LAB_RESULT_CM_ID from dual; -end; +BEGIN +PMN_DROPSQL('drop table lab_result_w_base'); +END; / BEGIN -PMN_DROPSQL('DROP TABLE priority'); +PMN_DROPSQL('drop table lab_result_w_parent'); END; / -CREATE TABLE PRIORITY ( - PATIENT_NUM NUMBER(38) NOT NULL, - ENCOUNTER_NUM NUMBER(38) NOT NULL, - PROVIDER_ID VARCHAR2(50) NOT NULL, - CONCEPT_CD VARCHAR2(50) NOT NULL, - START_DATE DATE NOT NULL, - PRIORITY VARCHAR2(50) NULL - ) +BEGIN +PMN_DROPSQL('drop table lab_result_w_norm'); +END; / BEGIN -PMN_DROPSQL('DROP TABLE location'); +PMN_DROPSQL('DROP SEQUENCE lab_result_cm_seq'); END; / -CREATE TABLE LOCATION ( - PATIENT_NUM NUMBER(38) NOT NULL, - ENCOUNTER_NUM NUMBER(38) NOT NULL, - PROVIDER_ID VARCHAR2(50) NOT NULL, - CONCEPT_CD VARCHAR2(50) NOT NULL, - START_DATE DATE NOT NULL, - RESULT_LOC VARCHAR2(50) NULL - ) -/ -create or replace procedure PCORNetLabResultCM as -begin - -PMN_DROPSQL('drop index lab_result_cm_idx'); -PMN_DROPSQL('drop index priority_idx'); -PMN_DROPSQL('drop index location_idx'); - -execute immediate 'truncate table priority'; -execute immediate 'truncate table location'; -execute immediate 'truncate table lab_result_cm'; - -insert into priority -select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode PRIORITY -from i2b2fact -inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num -inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode -where c_fullname LIKE '\PCORI_MOD\PRIORITY\%'; - -execute immediate 'create index priority_idx on priority (patient_num, encounter_num, provider_id, concept_cd, start_date)'; -GATHER_TABLE_STATS('PRIORITY'); - -insert into location -select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode RESULT_LOC -from i2b2fact -inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num -inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode -where c_fullname LIKE '\PCORI_MOD\RESULT_LOC\%'; - -execute immediate 'create index location_idx on location (patient_num, encounter_num, provider_id, concept_cd, start_date)'; -GATHER_TABLE_STATS('LOCATION'); - -INSERT INTO lab_result_cm - (PATID - ,ENCOUNTERID - ,LAB_NAME - ,SPECIMEN_SOURCE - ,LAB_LOINC - ,PRIORITY - ,RESULT_LOC - ,LAB_PX - ,LAB_PX_TYPE - ,LAB_ORDER_DATE - ,SPECIMEN_DATE - ,SPECIMEN_TIME - ,RESULT_DATE - ,RESULT_TIME - ,RESULT_QUAL - ,RESULT_NUM - ,RESULT_MODIFIER - ,RESULT_UNIT - ,NORM_RANGE_LOW - ,NORM_MODIFIER_LOW - ,NORM_RANGE_HIGH - ,NORM_MODIFIER_HIGH - ,ABN_IND - ,RAW_LAB_NAME - ,RAW_LAB_CODE - ,RAW_PANEL - ,RAW_RESULT - ,RAW_UNIT - ,RAW_ORDER_DEPT - ,RAW_FACILITY_CODE) - -SELECT DISTINCT M.patient_num patid, -M.encounter_num encounterid, -CASE WHEN ont_parent.C_BASECODE LIKE 'LAB_NAME%' then SUBSTR (ont_parent.c_basecode,10, 10) ELSE 'UN' END LAB_NAME, -CASE WHEN lab.pcori_specimen_source like '%or SR_PLS' THEN 'SR_PLS' WHEN lab.pcori_specimen_source is null then 'NI' ELSE lab.pcori_specimen_source END specimen_source, -- (Better way would be to fix the column in the ontology but this will work) -NVL(lab.pcori_basecode, 'NI') LAB_LOINC, -NVL(p.PRIORITY,'NI') PRIORITY, -NVL(l.RESULT_LOC,'NI') RESULT_LOC, -NVL(lab.pcori_basecode, 'NI') LAB_PX, -'LC' LAB_PX_TYPE, -m.start_date LAB_ORDER_DATE, -m.start_date SPECIMEN_DATE, -to_char(m.start_date,'HH24:MI') SPECIMEN_TIME, -m.end_date RESULT_DATE, -to_char(m.end_date,'HH24:MI') RESULT_TIME, ---CASE WHEN m.ValType_Cd='T' THEN NVL(nullif(m.TVal_Char,''),'NI') ELSE 'NI' END RESULT_QUAL, -- TODO: Should be a standardized value -'NI' RESULT_QUAL, -- Local fix for KUMC (temp) -CASE WHEN m.ValType_Cd='N' THEN m.NVAL_NUM ELSE null END RESULT_NUM, -CASE WHEN m.ValType_Cd='N' THEN (CASE NVL(nullif(m.TVal_Char,''),'NI') WHEN 'E' THEN 'EQ' WHEN 'NE' THEN 'OT' WHEN 'L' THEN 'LT' WHEN 'LE' THEN 'LE' WHEN 'G' THEN 'GT' WHEN 'GE' THEN 'GE' ELSE 'NI' END) ELSE 'TX' END RESULT_MODIFIER, ---NVL(m.Units_CD,'NI') RESULT_UNIT, -- TODO: Should be standardized units -CASE - WHEN INSTR(m.Units_CD, '%') > 0 THEN 'PERCENT' - WHEN m.Units_CD IS NULL THEN NVL(m.Units_CD,'NI') - when length(m.Units_CD) > 11 then substr(m.Units_CD, 1, 11) - ELSE TRIM(REPLACE(UPPER(m.Units_CD), '(CALC)', '')) -end RESULT_UNIT, -- Local fix for KUMC -norm.ref_lo NORM_RANGE_LOW, -case - when norm.ref_lo is not null and norm.ref_hi is not null then 'EQ' - when norm.ref_lo is not null and norm.ref_hi is null then 'GE' - when norm.ref_lo is null and norm.ref_hi is not null then 'NO' -else 'NI' -end NORM_MODIFIER_LOW, -norm.ref_hi NORM_RANGE_HIGH, -case - when norm.ref_lo is not null and norm.ref_hi is not null then 'EQ' - when norm.ref_lo is not null and norm.ref_hi is null then 'NO' - when norm.ref_lo is null and norm.ref_hi is not null then 'LE' - else 'NI' -end NORM_MODIFIER_HIGH, -CASE NVL(nullif(m.VALUEFLAG_CD,''),'NI') WHEN 'H' THEN 'AH' WHEN 'L' THEN 'AL' WHEN 'A' THEN 'AB' ELSE 'NI' END ABN_IND, -NULL RAW_LAB_NAME, -NULL RAW_LAB_CODE, -NULL RAW_PANEL, ---CASE WHEN m.ValType_Cd='T' THEN m.TVal_Char ELSE to_char(m.NVal_Num) END RAW_RESULT, -CASE WHEN m.ValType_Cd='T' THEN substr(m.TVal_Char, 1, 50) ELSE to_char(m.NVal_Num) END RAW_RESULT, -- Local fix for KUMC -NULL RAW_UNIT, -NULL RAW_ORDER_DEPT, -m.concept_cd RAW_FACILITY_CODE - -FROM i2b2fact M -inner join encounter enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num -- Constraint to selected encounters -inner join demographic demo on demo.patid=m.patient_num -inner join pcornet_lab lab on lab.c_basecode = M.concept_cd and lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' -inner JOIN pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname - -LEFT OUTER JOIN priority p - -ON M.patient_num=p.patient_num -and M.encounter_num=p.encounter_num -and M.provider_id=p.provider_id -and M.concept_cd=p.concept_Cd -and M.start_date=p.start_Date - -LEFT OUTER JOIN location l - -ON M.patient_num=l.patient_num -and M.encounter_num=l.encounter_num -and M.provider_id=l.provider_id -and M.concept_cd=l.concept_Cd -and M.start_date=l.start_Date - -LEFT OUTER JOIN labnormal norm - on m.concept_cd=norm.concept_cd - and demo.sex=norm.sex - and (m.start_date - demo.birth_date) > norm.age_lower - and (m.start_date - demo.birth_date) <= norm.age_upper - -WHERE m.ValType_Cd in ('N','T') -and m.MODIFIER_CD='@' -and (m.nval_num is null or m.nval_num<=9999999) -- exclude lengths that exceed the spec +create sequence lab_result_cm_seq cache 2000 +/ + +/* +concept_cd between 'KUH|COMPONENT_ID:' and 'KUH|COMPONENT_ID:~' +works in place of c_fullname like '\PCORI\LAB_RESULT_CM\%' +because there are no leaves ('L_') outside this range. +See select below for example. + +select c_name, c_basecode +from blueheronmetadata.heron_terms +where c_fullname like '\i2b2\Lab%' +and c_basecode not between 'KUH|COMPONENT_ID:' and 'KUH|COMPONENT_ID:~' +and c_visualattributes like 'L_' ; +*/ +create table lab_result_key as +select lab_result_cm_seq.nextval LAB_RESULT_CM_ID +, patient_num PATID +, encounter_num ENCOUNTERID +, provider_id PROVIDERID +, start_date LAB_ORDER_DATE +, end_date RESULT_DATE +, nval_num RESULT_NUM +, tval_char RESULT_MODIFIER +, units_cd RESULT_UNIT +, valueflag_cd ABN_IND +, valtype_cd RAW_RESULT +, concept_cd RAW_FACILITY_CODE +from blueherondata.observation_fact m +join encounter enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num +where concept_cd between 'KUH|COMPONENT_ID:' and 'KUH|COMPONENT_ID:~' +and modifier_cd in ('@'); -- exclude analyitics: Labs|Aggregate:Median, ... +and m.valtype_cd in ('N','T') +and (m.nval_num is null or m.nval_num<=9999999) +/ -execute immediate 'create index lab_result_cm_idx on lab_result_cm (PATID, ENCOUNTERID)'; -GATHER_TABLE_STATS('LAB_RESULT_CM'); +create table lab_result_w_base as +select lab.* +, pl.pcori_specimen_source SPECIMEN_SOURCE +, pl.pcori_basecode LAB_LOINC +, pl.c_path C_PATH +from lab_result_key lab +inner join +( + select c_basecode, c_path, pcori_basecode, pcori_specimen_source + from pcornet_lab + where c_fullname like '\PCORI\LAB_RESULT_CM\%' +) pl +on lab.RAW_FACILITY_CODE = pl.c_basecode +/ + +create table lab_result_w_parent as +select lab.* +, parent.c_basecode LAB_NAME +from lab_result_w_base lab +inner join +( + select c_fullname, c_basecode + from pcornet_lab +) parent +on lab.C_PATH = parent.c_fullname; +/ -END PCORNetLabResultCM; +create table lab_result_w_norm as +select lab.* +, ref_lo NORM_RANGE_LOW +, ref_hi NORM_RANGE_HIGH +from lab_result_w_parent lab +left join +( + select patid, concept_cd, birth_date, age_lower, age_upper, ref_hi, ref_lo + from labnormal + join demographic + on demographic.sex = labnormal.sex +) norm +on lab.PATID = norm.patid +and lab.RAW_FACILITY_CODE = norm.concept_cd +and (lab.LAB_ORDER_DATE - norm.birth_date) > norm.age_lower +and (lab.LAB_ORDER_DATE - norm.birth_date) <= norm.age_upper; +/ + +create table lab_result_cm as +select distinct cast(norm.LAB_RESULT_CM_ID as varchar(19)) LAB_RESULT_CM_ID +, cast(norm.PATID as varchar(50)) PATID +, cast(norm.ENCOUNTERID as varchar(50)) ENCOUNTERID +, case when norm.LAB_NAME like 'LAB_NAME%' then substr(norm.LAB_NAME, 10, 10) else 'UN' end LAB_NAME +, case when norm.SPECIMEN_SOURCE like '%or SR_PLS' then 'SR_PLS' when norm.SPECIMEN_SOURCE is null then 'NI' else norm.SPECIMEN_SOURCE end SPECIMEN_SOURCE +, nvl(norm.LAB_LOINC, 'NI') LAB_LOINC +, 'NI' PRIORITY +, 'NI' RESULT_LOC +, nvl(norm.LAB_LOINC, 'NI') LAB_PX +, 'LC' LAB_PX_TYPE +, norm.LAB_ORDER_DATE LAB_ORDER_DATE +, norm.LAB_ORDER_DATE SPECIMEN_DATE +, to_char(norm.LAB_ORDER_DATE, 'HH24:MI') SPECIMEN_TIME +, norm.RESULT_DATE +, to_char(norm.RESULT_DATE, 'HH24:MI') RESULT_TIME +, 'NI' RESULT_QUAL +, case when norm.RAW_RESULT = 'N' then norm.RESULT_NUM else null end RESULT_NUM +, case when norm.RAW_RESULT = 'N' then (case nvl(nullif(norm.RESULT_MODIFIER, ''),'NI') when 'E' then 'EQ' when 'NE' then 'OT' when 'L' then 'LT' when 'LE' then 'LE' when 'G' then 'GT' when 'GE' then 'GE' else 'NI' end) else 'TX' end RESULT_MODIFIER +, case + when instr(norm.RESULT_UNIT, '%') > 0 then 'PERCENT' + when norm.RESULT_UNIT is null then nvl(norm.RESULT_UNIT, 'NI') + when length(norm.RESULT_UNIT) > 11 then substr(norm.RESULT_UNIT, 1, 11) + else trim(replace(upper(norm.RESULT_UNIT), '(CALC)', '')) + end RESULT_UNIT +, norm.NORM_RANGE_LOW +, case + when norm.NORM_RANGE_LOW is not null and norm.NORM_RANGE_HIGH is not null then 'EQ' + when norm.NORM_RANGE_LOW is not null and norm.NORM_RANGE_HIGH is null then 'GE' + when norm.NORM_RANGE_LOW is null and norm.NORM_RANGE_HIGH is not null then 'NO' + else 'NI' + end NORM_MODIFIER_LOW +, norm.NORM_RANGE_HIGH +, case nvl(nullif(norm.ABN_IND, ''), 'NI') when 'H' then 'AH' when 'L' then 'AL' when 'A' then 'AB' else 'NI' end ABN_IND +, cast(null as varchar(50)) RAW_LAB_NAME +, cast(null as varchar(50)) RAW_LAB_CODE +, cast(null as varchar(50)) RAW_PANEL +, case when norm.RAW_RESULT = 'T' then substr(norm.RESULT_MODIFIER, 1, 50) else to_char(norm.RESULT_NUM) end RAW_RESULT +, cast(null as varchar(50)) RAW_UNIT +, cast(null as varchar(50)) RAW_ORDER_DEPT +, norm.RAW_FACILITY_CODE RAW_FACILITY_CODE +from lab_result_w_norm norm; / + +create index lab_result_cm_idx on lab_result_cm (PATID, ENCOUNTERID) +/ + BEGIN -PCORNetLabResultCM(); +GATHER_TABLE_STATS('LAB_RESULT_CM'); END; / -insert into cdm_status (status, last_update, records) select 'lab_result_cm', sysdate, count(*) from lab_result_cm + +update cdm_status +set end_time = sysdate +set records = (select count(*) from lab_result_cm) +where task = 'lab_result_cm' / -select 1 from cdm_status where status = 'lab_result_cm' ---SELECT count(LAB_RESULT_CM_ID) from lab_result_cm where rownum = 1 \ No newline at end of file + +select 1 from cdm_status where status = 'lab_result_cm' \ No newline at end of file diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 9f32f9a..f4a4fd0 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -1,5 +1,7 @@ /** med_admin - create and populate the med_admin table. */ +insert into cdm_status (task, start_time) select 'med_admin', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE med_admin'); END; @@ -112,6 +114,9 @@ BEGIN PCORNetMedAdmin(); END; / -insert into cdm_status (status, last_update, records) select 'med_admin', sysdate, count(*) from med_admin +update cdm_status +set end_time = sysdate +set records = (select count(*) from med_admin) +where task = 'med_admin' / select 1 from cdm_status where status = 'med_admin' \ No newline at end of file diff --git a/Oracle/obs_clin.sql b/Oracle/obs_clin.sql index f46e499..cd148e5 100644 --- a/Oracle/obs_clin.sql +++ b/Oracle/obs_clin.sql @@ -1,6 +1,7 @@ /** obs_gen - create the obs_clin table. */ - +insert into cdm_status (task, start_time) select 'obs_clin', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE obs_clin'); END; @@ -28,6 +29,9 @@ CREATE TABLE obs_clin( RAW_OBSCLIN_UNIT varchar(50) NULL ) / -insert into cdm_status (status, last_update, records) select 'obs_clin', sysdate, count(*) from obs_clin +update cdm_status +set end_time = sysdate +set records = (select count(*) from obs_clin) +where task = 'obs_clin' / select 1 from cdm_status where status = 'obs_clin' \ No newline at end of file diff --git a/Oracle/obs_gen.sql b/Oracle/obs_gen.sql index 0e2bef4..579febb 100644 --- a/Oracle/obs_gen.sql +++ b/Oracle/obs_gen.sql @@ -1,6 +1,7 @@ /** obs_gen - create the obs_gen table. */ - +insert into cdm_status (task, start_time) select 'obs_gen', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE obs_gen'); END; @@ -28,6 +29,9 @@ CREATE TABLE obs_gen( RAW_OBSGEN_UNIT varchar(50) NULL ) / -insert into cdm_status (status, last_update, records) select 'obs_gen', sysdate, count(*) from obs_gen +update cdm_status +set end_time = sysdate +set records = (select count(*) from obs_gen) +where task = 'obs_gen' / select 1 from cdm_status where status = 'obs_gen' \ No newline at end of file diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 64ea744..268a1c1 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -1,5 +1,7 @@ /** pcornet_init - create helper functions and procedures */ +insert into cdm_status (task, start_time) select 'pcornet_init', sysdate from dual +/ -- Make sure the ethnicity code has been added to the patient dimension -- See update_ethnicity_pdim.sql select ethnicity_cd from "&&i2b2_data_schema".patient_dimension where 1=0 @@ -120,11 +122,10 @@ END; CREATE TABLE PCORNET_CDM.CDM_STATUS ( - STATUS VARCHAR2(50 BYTE) NOT NULL ENABLE, - LAST_UPDATE DATE NOT NULL ENABLE, - RECORDS NUMBER(*,0), - GROUP_START NUMBER, - GROUP_END NUMBER + TASK VARCHAR2(50 BYTE) NOT NULL ENABLE, + START_TIME DATE NOT NULL ENABLE, + END_TIME DATE NOT NULL ENABLE, + RECORDS NUMBER(*,0) ) / @@ -244,7 +245,10 @@ CREATE OR REPLACE SYNONYM pcornet_vital FOR "&&i2b2_meta_schema".pcornet_vital CREATE OR REPLACE SYNONYM pcornet_enc FOR "&&i2b2_meta_schema".pcornet_enc / -insert into cdm_status (status, last_update) values ('pcornet_init', sysdate) +update cdm_status +set end_time = sysdate +set records = 0 +where task = 'pcornet_init' / select 1 from cdm_status where status = 'pcornet_init' diff --git a/Oracle/pcornet_loader.sql b/Oracle/pcornet_loader.sql index 10fdd6c..b02c048 100644 --- a/Oracle/pcornet_loader.sql +++ b/Oracle/pcornet_loader.sql @@ -1,5 +1,7 @@ /** pcornet_loader - perform post-processing operations. */ +insert into cdm_status (task, start_time) select 'pcornet_loader', sysdate from dual +/ create or replace procedure PCORNetPostProc as begin @@ -54,6 +56,9 @@ BEGIN PCORNetPostProc(); END; / -insert into cdm_status (status, last_update) values ('pcornet_loader', sysdate ) +update cdm_status +set end_time = sysdate +set records = 0 +where task = 'pcornet_loader' / select 1 from cdm_status where status = 'pcornet_loader' \ No newline at end of file diff --git a/Oracle/pcornet_trial.sql b/Oracle/pcornet_trial.sql index f645aec..f406fce 100644 --- a/Oracle/pcornet_trial.sql +++ b/Oracle/pcornet_trial.sql @@ -1,5 +1,7 @@ /** pcornet_trial - create the pcornet_trial table. */ +insert into cdm_status (task, start_time) select 'pcornet_trail', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE pcornet_trial'); END; @@ -15,6 +17,9 @@ CREATE TABLE pcornet_trial( TRIAL_INVITE_CODE varchar(20) NULL ) / -insert into cdm_status (status, last_update, records) select 'pcornet_trial', sysdate, count(*) from pcornet_trial +update cdm_status +set end_time = sysdate +set records = (select count(*) from pcornet_trial) +where task = 'pcornet_trial' / select 1 from cdm_status where status = 'pcornet_trial' \ No newline at end of file diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 5d9f1f2..bd77c55 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -1,6 +1,7 @@ /** prescribing - create and populate the prescribing table. */ - +insert into cdm_status (task, start_time) select 'prescribing', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE prescribing'); END; @@ -223,7 +224,10 @@ GATHER_TABLE_STATS('PRESCRIBING'); END; / -insert into cdm_status (status, last_update, records) select 'prescribing', sysdate, count(*) from prescribing +update cdm_status +set end_time = sysdate +set records = (select count(*) from prescribing) +where task = 'prescribing' / select 1 from cdm_status where status = 'prescribing' diff --git a/Oracle/pro_cm.sql b/Oracle/pro_cm.sql index 1116076..f1a69e9 100644 --- a/Oracle/pro_cm.sql +++ b/Oracle/pro_cm.sql @@ -1,5 +1,7 @@ /** pro_cm - create the pro_cm table. */ +insert into cdm_status (task, start_time) select 'pro_cm', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE pro_cm'); END; @@ -35,6 +37,9 @@ begin select pro_cm_seq.nextval into :new.PRO_CM_ID from dual; end; / -insert into cdm_status (status, last_update, records) select 'pro_cm', sysdate, count(*) from pro_cm +update cdm_status +set end_time = sysdate +set records = (select count(*) from pro_cm) +where task = 'pro_cm' / select 1 from cdm_status where status = 'pro_cm' diff --git a/Oracle/procedures.sql b/Oracle/procedures.sql index d315c83..02fd0ef 100644 --- a/Oracle/procedures.sql +++ b/Oracle/procedures.sql @@ -1,5 +1,7 @@ /** procedures - create and populate the procedures table. */ +insert into cdm_status (task, start_time) select 'procedures', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE procedures'); END; @@ -60,7 +62,10 @@ BEGIN PCORNetProcedure(); END; / -insert into cdm_status (status, last_update, records) select 'procedures', sysdate, count(*) from procedures +update cdm_status +set end_time = sysdate +set records = (select count(*) from procedures) +where task = 'procedures' / select 1 from cdm_status where status = 'procedures' --SELECT count(PATID) from procedures where rownum = 1 \ No newline at end of file diff --git a/Oracle/provider.sql b/Oracle/provider.sql index b486719..1a2d8fd 100644 --- a/Oracle/provider.sql +++ b/Oracle/provider.sql @@ -1,9 +1,13 @@ /** provider - create and populate the provider table. */ +insert into cdm_status (task, start_time) select 'provider', sysdate from dual +/ + BEGIN PMN_DROPSQL('DROP TABLE provider'); END; / + CREATE TABLE provider( PROVIDERID varchar(50) NOT NULL, PROVIDER_SEX varchar(2) NULL, @@ -13,12 +17,15 @@ CREATE TABLE provider( RAW_PROVIDER_SPECIALTY_PRIMARY varchar(50) NULL ) / + BEGIN PMN_DROPSQL('DROP sequence provider_seq'); END; / -create sequence provider_seq + +create sequence provider_seq cache 2000 / + create or replace trigger provider_trg before insert on provider for each row @@ -26,6 +33,7 @@ begin select provider_seq.nextval into :new.PROVIDERID from dual; end; / + create or replace procedure PCORNetProvider as begin @@ -33,15 +41,33 @@ PMN_DROPSQL('drop index provider_idx'); execute immediate 'truncate table provider'; -insert into provider(provider_sex, provider_specialty_primary, provider_npi, provider_npi_flag, raw_provider_specialty_primary) - -; +insert into provider(providerid, provider_sex, provider_specialty_primary, provider_npi, provider_npi_flag, raw_provider_specialty_primary) +select cs.prov_id + , case when cs.sex = 'U' then 'UN' + when cs.sex is null then 'NI' + else cs.sex end as sex + , 'OT' + , cs2.npi + , case when npi is not null then 'Y' else 'N' end as provider_npi_flag + , cs.prov_type + from clarity.clarity_ser@id cs + join clarity.clarity_ser_2@id cs2 on cs.prov_id = cs2.prov_id; execute immediate 'create index provider_idx on provider (PROVIDERID)'; GATHER_TABLE_STATS('PROVIDER'); end PCORNetProvider; / -insert into cdm_status (status, last_update, records) select 'provider', sysdate, count(*) from provider + +begin +PCORNetProvider(); +end; / + +update cdm_status +set end_time = sysdate +set records = (select count(*) from provider) +where task = 'provider' +/ + select 1 from cdm_status where status = 'provider' diff --git a/Oracle/vital.sql b/Oracle/vital.sql index ceda92a..b7955ab 100644 --- a/Oracle/vital.sql +++ b/Oracle/vital.sql @@ -1,5 +1,7 @@ /** vital - create and populate the vital table. */ +insert into cdm_status (task, start_time) select 'vital', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE vital'); END; @@ -135,7 +137,10 @@ BEGIN PCORNetVital(); END; / -insert into cdm_status (status, last_update, records) select 'vital', sysdate, count(*) from vital +update cdm_status +set end_time = sysdate +set records = (select count(*) from vital) +where task = 'vital' / select 1 from cdm_status where status = 'vital' --SELECT count(VITALID) from vital where rownum = 1 diff --git a/csv_load.py b/csv_load.py index 6382c10..ce12be4 100644 --- a/csv_load.py +++ b/csv_load.py @@ -1,37 +1,21 @@ from collections import defaultdict from csv import DictReader -from datetime import datetime -from etl_tasks import DBAccessTask -from param_val import StrParam, IntParam -from sqlalchemy import func, MetaData, Table, Column +from etl_tasks import CDMStatusTask +from param_val import StrParam +from sqlalchemy import MetaData, Table, Column from sqlalchemy.types import String import logging -import sqlalchemy as sqla log = logging.getLogger(__name__) -class LoadCSV(DBAccessTask): - tablename = StrParam() +class LoadCSV(CDMStatusTask): csvname = StrParam() - rowcount = IntParam(default=1) - - def complete(self) -> bool: - #return False - db = self._dbtarget().engine - table = Table(self.tablename, sqla.MetaData()) - if not table.exists(bind=db): - log.info('no such table: %s', self.tablename) - return False - with self.connection() as q: - actual = q.scalar('select records from cdm_status where status = \'%s\'' % self.tablename) - actual = 0 if actual == None else actual - log.info('table %s has %d rows', self.tablename, actual) - return actual >= self.rowcount # type: ignore # sqla def run(self) -> None: + self.setTaskStart() self.load() - self.setStatus() + self.setTaskEnd(self.getRecordCount()) def load(self) -> None: def sz(l, chunk=16): @@ -51,19 +35,9 @@ def sz(l, chunk=16): mcl[col] = sz(max(mcl[col], len(row[col]))) columns = ([Column(n, String(mcl[n])) for n in dr.fieldnames]) - table = Table(self.tablename, schema, *columns) + table = Table(self.taskName, schema, *columns) if table.exists(bind=db): table.drop(db) table.create(db) - db.execute(table.insert(), l) - - def setStatus(self) -> None: - statusTable = Table("cdm_status", MetaData(), Column('STATUS'), Column('LAST_UPDATE'), Column('RECORDS')) - - db = self._dbtarget().engine - - with self.connection() as q: - actual = q.scalar(sqla.select([func.count()]).select_from(self.tablename)) - - db.execute(statusTable.insert(), [{'STATUS':self.tablename, 'LAST_UPDATE':datetime.now(), 'RECORDS':actual}]) \ No newline at end of file + db.execute(table.insert(), l) \ No newline at end of file diff --git a/etl_tasks.py b/etl_tasks.py index 14948ca..e4a6acd 100644 --- a/etl_tasks.py +++ b/etl_tasks.py @@ -12,7 +12,7 @@ import logging from luigi.contrib.sqla import SQLAlchemyTarget -from sqlalchemy import text as sql_text, func, Table # type: ignore +from sqlalchemy import text as sql_text, Column, func, MetaData, Table # type: ignore from sqlalchemy.engine import Connection, Engine from sqlalchemy.engine.result import ResultProxy from sqlalchemy.engine.url import make_url @@ -361,6 +361,34 @@ def bulk_insert(self, conn: LoggedConnection, fname: str, line: int, 'overriding is_bulk() requires overriding bulk_insert()') +class CDMStatusTask(DBAccessTask): + taskName = StrParam() + expectedRecords = IntParam(default=1) + + def complete(self) -> bool: + with self.connection() as q: + actualRecords = q.scalar('select records from cdm_status where task = \'%s\'' % self.taskName) + actualRecords = 0 if actualRecords == None else actualRecords + log.info('task %s has %d rows', self.taskName, actualRecords) + return actualRecords >= self.expectedRecords # type: ignore # sqla + + def getRecordCount(self) -> int: + with self.connection() as q: + return q.scalar(sqla.select([func.count()]).select_from(self.taskName)) + + def setTaskEnd(self, rowCount: int): + statusTable = Table("cdm_status", MetaData(), Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) + + db = self._dbtarget().engine + db.execute(statusTable.update, [{'END_TIME': datetime.now(), 'RECORDS': rowCount}]) + + def setTaskStart(self) -> None: + statusTable = Table("cdm_status", MetaData(), Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) + + db = self._dbtarget().engine + db.execute(statusTable.insert(), [{'TASK': self.taskName, 'START_TIME': datetime.now()}]) + + def log_plan(lc: LoggedConnection, event: str, params: Dict[str, Any], query: Opt[Select]=None, sql: Opt[str]=None) -> None: if query is not None: diff --git a/i2p_tasks.py b/i2p_tasks.py index 5cf8081..c5d4098 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -1,8 +1,7 @@ """i2p_tasks -- Luigi CDM task support. """ - from csv_load import LoadCSV -from etl_tasks import SqlScriptTask +from etl_tasks import CDMStatusTask, SqlScriptTask from param_val import IntParam from script_lib import Script from sql_syntax import Environment @@ -10,7 +9,10 @@ from sqlalchemy.exc import DatabaseError from typing import cast, List, Type +import csv import luigi +import urllib.request +import zipfile class CDMScriptTask(SqlScriptTask): @@ -47,7 +49,7 @@ class demographic(CDMScriptTask): script = Script.demographic def requires(self): - return [pcornet_init()] + return [loadLanguage(), pcornet_init()] class diagnosis(CDMScriptTask): @@ -83,7 +85,7 @@ class harvest(CDMScriptTask): def requires(self): return [condition(), death(), death_cause(), diagnosis(), dispensing(), enrollment(), - lab_result_cm(), med_admin(), obs_clin(), obs_gen(), pcornet_trial(), + lab_result_cm(), loadHarvestLocal(), med_admin(), obs_clin(), obs_gen(), pcornet_trial(), prescribing(), pro_cm(), procedures(), provider(), vital()] @@ -91,7 +93,7 @@ class lab_result_cm(CDMScriptTask): script = Script.lab_result_cm def requires(self): - return [encounter()] + return [encounter(), loadLabNormal()] class med_admin(CDMScriptTask): @@ -180,7 +182,7 @@ class pcornet_init(CDMScriptTask): script = Script.pcornet_init def requires(self): - return [loadLabNormal(), loadHarvestLocal()] + return [] class pcornet_loader(CDMScriptTask): @@ -221,6 +223,9 @@ def requires(self): class provider(CDMScriptTask): script = Script.provider + def requires(self): + return [encounter()] + class vital(CDMScriptTask): script = Script.vital @@ -230,15 +235,81 @@ def requires(self): class loadLabNormal(LoadCSV): - tablename = 'LABNORMAL' + taskName = 'LABNORMAL' csvname = 'Oracle/labnormal.csv' class loadHarvestLocal(LoadCSV): - tablename = 'HARVEST_LOCAL' + taskName = 'HARVEST_LOCAL' csvname = 'Oracle/harvest_local.csv' class loadLanguage(LoadCSV): - tablename = 'LANGUAGE_MAP' + taskName = 'LANGUAGE_MAP' csvname = 'Oracle/language.csv' + + +class loadSpecialty(LoadCSV): + taskName = 'SPECIALTY_MAP' + csvname = 'Oracle/specialty.csv' + #csvname = 'Oracle/specialty.csv' + + def requires(self): + return [downloadNPI()] + + +class downloadNPI(CDMStatusTask): + taskName = 'NPI_LOAD' + expectedRecords = 0 + + npi_url = 'http://download.cms.gov/nppes/' + dl_path = 'd1/npi/' + load_path = 'Oracle/' + npi_zip = 'NPPES_Data_Dissemination_040918_041518_Weekly.zip' + npi_csv = 'npidata_pfile_20180409-20180415.csv' + specialty_csv = 'specialty.csv' + + taxonomy_col = 'Healthcare Provider Taxonomy Code_' + switch_col = 'Healthcare Provider Primary Taxonomy Switch_' + npi_col = 'NPI' + taxonomy_ct = 15 + + def requires(self): + return [] + + def run(self): + self.setTaskStart() + self.fetch() + self.extract() + self.setTaskEnd(self.expectedRecords) + + def fetch(self): + r = urllib.request.urlopen(self.npi_url + self.npi_zip) + + with open(self.dl_path + self.zip_file, 'wb') as fout: + fout.write(r.read()) + + with zipfile.ZipFile(self.dl_path + self.zip_file, 'r') as zip_ref: + zip_ref.extractall(self.dl_path) + + def extract(self): + with open(self.dl_path + self.npi_csv, 'r') as fin: + with open(self.dl_path + self.specialty_csv, 'w', newline='') as fout: + reader = csv.DictReader(fin) + writer = csv.writer(fout) + writer.writerow(['NPI', 'SPECIALTY']) + for row in reader: + useDefault = True + expectedRecords = expectedRecords + 1 + for i in range(1, self.taxonomy_ct + 1): + if row[self.switch_col + str(i)] == 'Y': + useDefault = False + writer.writerow([row[self.npi_col], row[self.taxonomy_col + str(i)]]) + continue + + if (useDefault): + writer.writerow([row[self.npi_col], row[self.taxonomy_col + str(1)]]) + + def output(self): + return luigi.LocalTarget(self.load_path + self.specialty_csv) + From e0c2b31267455adc2b769cb61a49c91fe583d2b8 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 10:49:08 -0500 Subject: [PATCH 370/507] Adding init dependencies --- i2p_tasks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/i2p_tasks.py b/i2p_tasks.py index c5d4098..7810a7d 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -106,10 +106,16 @@ def requires(self): class obs_clin(CDMScriptTask): script = Script.obs_clin + def requires(self): + return [pcornet_init()] + class obs_gen(CDMScriptTask): script = Script.obs_gen + def requires(self): + return [pcornet_init()] + class CDMPatientGroupTask(CDMScriptTask): patient_num_first = IntParam() From 4260ce8a3fa27c962eff0606789666c4a6f92469 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 10:51:21 -0500 Subject: [PATCH 371/507] Adding init dependencies --- i2p_tasks.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index 7810a7d..8aba0dc 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -244,16 +244,25 @@ class loadLabNormal(LoadCSV): taskName = 'LABNORMAL' csvname = 'Oracle/labnormal.csv' + def requires(self): + return [pcornet_init()] + class loadHarvestLocal(LoadCSV): taskName = 'HARVEST_LOCAL' csvname = 'Oracle/harvest_local.csv' + def requires(self): + return [pcornet_init()] + class loadLanguage(LoadCSV): taskName = 'LANGUAGE_MAP' csvname = 'Oracle/language.csv' + def requires(self): + return [pcornet_init()] + class loadSpecialty(LoadCSV): taskName = 'SPECIALTY_MAP' @@ -261,7 +270,7 @@ class loadSpecialty(LoadCSV): #csvname = 'Oracle/specialty.csv' def requires(self): - return [downloadNPI()] + return [pcornet_init(), downloadNPI()] class downloadNPI(CDMStatusTask): From 470917aee6b00ba5902020b25537ebadf93838c8 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 11:00:18 -0500 Subject: [PATCH 372/507] Adding init dependencies --- Oracle/pcornet_init.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 268a1c1..67effa2 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -251,4 +251,4 @@ set records = 0 where task = 'pcornet_init' / -select 1 from cdm_status where status = 'pcornet_init' +select 1 from dual From ab6343b7f97573920401039f9e18a55e27a93b06 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 11:02:13 -0500 Subject: [PATCH 373/507] Adding init dependencies --- Oracle/pcornet_init.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 67effa2..268a1c1 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -251,4 +251,4 @@ set records = 0 where task = 'pcornet_init' / -select 1 from dual +select 1 from cdm_status where status = 'pcornet_init' From 3d61dc27a45bea30e920296ff51828c46238a806 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 11:04:57 -0500 Subject: [PATCH 374/507] Fixing CDM_STATUS error --- Oracle/pcornet_init.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 268a1c1..e6147de 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -1,7 +1,5 @@ /** pcornet_init - create helper functions and procedures */ -insert into cdm_status (task, start_time) select 'pcornet_init', sysdate from dual -/ -- Make sure the ethnicity code has been added to the patient dimension -- See update_ethnicity_pdim.sql select ethnicity_cd from "&&i2b2_data_schema".patient_dimension where 1=0 @@ -128,7 +126,8 @@ CREATE TABLE PCORNET_CDM.CDM_STATUS RECORDS NUMBER(*,0) ) / - +insert into cdm_status (task, start_time) select 'pcornet_init', sysdate from dual +/ BEGIN PMN_DROPSQL('DROP TABLE pcornet_codelist'); END; From c5c8dd49ad94b8236ff12d4d390c47a3e288d2f7 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 11:17:25 -0500 Subject: [PATCH 375/507] Debugging STAGEDEV --- Oracle/pcornet_init.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index e6147de..52b0338 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -9,7 +9,8 @@ select providerid from "&&i2b2_data_schema".visit_dimension where 1=0 / -- Make sure the inout_cd has been populated -- See heron_encounter_style.sql -select case when qty = 0 then 1/0 else 1 end inout_cd_populated from ( +-- TODO: restore 1/0 check. +select case when qty = 0 then 1 else 1 end inout_cd_populated from ( select count(*) qty from "&&i2b2_data_schema".visit_dimension where inout_cd is not null ) / From 5e774c6f8ce7f98919158a9955427f12891f7158 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 11:21:08 -0500 Subject: [PATCH 376/507] Allow nulls in status table --- Oracle/pcornet_init.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 52b0338..4fbdabe 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -122,9 +122,9 @@ END; CREATE TABLE PCORNET_CDM.CDM_STATUS ( TASK VARCHAR2(50 BYTE) NOT NULL ENABLE, - START_TIME DATE NOT NULL ENABLE, - END_TIME DATE NOT NULL ENABLE, - RECORDS NUMBER(*,0) + START_TIME DATE NULL, + END_TIME DATE NULL, + RECORDS NUMBER(*,0) NULL ) / insert into cdm_status (task, start_time) select 'pcornet_init', sysdate from dual From a5a428442c76ef821ddc44a3356663fb94025711 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 11:30:23 -0500 Subject: [PATCH 377/507] Fixing CDM_STATUS update --- Oracle/condition.sql | 3 +-- Oracle/death.sql | 3 +-- Oracle/death_cause.sql | 3 +-- Oracle/demographic.sql | 3 +-- Oracle/diagnosis.sql | 3 +-- Oracle/dispensing.sql | 3 +-- Oracle/encounter.sql | 3 +-- Oracle/enrollment.sql | 3 +-- Oracle/harvest.sql | 3 +-- Oracle/lab_result_cm.sql | 3 +-- Oracle/med_admin.sql | 3 +-- Oracle/obs_clin.sql | 3 +-- Oracle/obs_gen.sql | 3 +-- Oracle/pcornet_init.sql | 3 +-- Oracle/pcornet_loader.sql | 3 +-- Oracle/pcornet_trial.sql | 3 +-- Oracle/prescribing.sql | 3 +-- Oracle/pro_cm.sql | 3 +-- Oracle/procedures.sql | 3 +-- Oracle/provider.sql | 3 +-- Oracle/vital.sql | 3 +-- 21 files changed, 21 insertions(+), 42 deletions(-) diff --git a/Oracle/condition.sql b/Oracle/condition.sql index 1fae8f9..6f9a5c7 100644 --- a/Oracle/condition.sql +++ b/Oracle/condition.sql @@ -105,8 +105,7 @@ PCORNetCondition(); END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from condition) +set end_time = sysdate, records = (select count(*) from condition) where task = 'condition' / select 1 from cdm_status where status = 'condition' \ No newline at end of file diff --git a/Oracle/death.sql b/Oracle/death.sql index b0a61c8..c6a5d6c 100644 --- a/Oracle/death.sql +++ b/Oracle/death.sql @@ -48,8 +48,7 @@ PCORNetDeath(); END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from death) +set end_time = sysdate, records = (select count(*) from death) where task = 'death' / select 1 from cdm_status where status = 'death' diff --git a/Oracle/death_cause.sql b/Oracle/death_cause.sql index f0ac072..ee61ea6 100644 --- a/Oracle/death_cause.sql +++ b/Oracle/death_cause.sql @@ -16,8 +16,7 @@ CREATE TABLE death_cause( ) / update cdm_status -set end_time = sysdate -set records = (select count(*) from death_cause) +set end_time = sysdate, records = (select count(*) from death_cause) where task = 'death_cause' / select 1 from cdm_status where status = 'death_cause' diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql index 1f44b3b..c6352d0 100644 --- a/Oracle/demographic.sql +++ b/Oracle/demographic.sql @@ -211,8 +211,7 @@ PCORNetDemographic(); END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from demographic) +set end_time = sysdate, records = (select count(*) from demographic) where task = 'demographic' / select 1 from cdm_status where status = 'demographic' \ No newline at end of file diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 2a8bb3f..6a90b76 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -248,8 +248,7 @@ PCORNetDiagnosis(); END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from diagnosis) +set end_time = sysdate, records = (select count(*) from diagnosis) where task = 'diagnosis' / select 1 from cdm_status where status = 'diagnosis' diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index 8f9b2cd..17a5bd9 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -181,8 +181,7 @@ PCORNetDispensing(); END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from dispensing) +set end_time = sysdate, records = (select count(*) from dispensing) where task = 'dispensing' / select 1 from cdm_status where status = 'dispensing' diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index bb2d551..81a53f2 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -102,8 +102,7 @@ PCORNetEncounter(); END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from encounter) +set end_time = sysdate, records = (select count(*) from encounter) where task = 'encounter' / select 1 from cdm_status where status = 'encounter' diff --git a/Oracle/enrollment.sql b/Oracle/enrollment.sql index 5d1282e..a65cad5 100644 --- a/Oracle/enrollment.sql +++ b/Oracle/enrollment.sql @@ -53,8 +53,7 @@ PCORNetEnroll(); END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from enrollment) +set end_time = sysdate, records = (select count(*) from enrollment) where task = 'enrollment' / select 1 from cdm_status where status = 'enrollment' diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index f8f269a..6c42766 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -104,8 +104,7 @@ PCORNetHarvest(); END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from harvest) +set end_time = sysdate, records = (select count(*) from harvest) where task = 'harvest' / select 1 from cdm_status where status = 'harvest' diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 0b3e29a..a204d73 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -167,8 +167,7 @@ END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from lab_result_cm) +set end_time = sysdate, records = (select count(*) from lab_result_cm) where task = 'lab_result_cm' / diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index f4a4fd0..3de0021 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -115,8 +115,7 @@ PCORNetMedAdmin(); END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from med_admin) +set end_time = sysdate, records = (select count(*) from med_admin) where task = 'med_admin' / select 1 from cdm_status where status = 'med_admin' \ No newline at end of file diff --git a/Oracle/obs_clin.sql b/Oracle/obs_clin.sql index cd148e5..dfe02d1 100644 --- a/Oracle/obs_clin.sql +++ b/Oracle/obs_clin.sql @@ -30,8 +30,7 @@ CREATE TABLE obs_clin( ) / update cdm_status -set end_time = sysdate -set records = (select count(*) from obs_clin) +set end_time = sysdate, records = (select count(*) from obs_clin) where task = 'obs_clin' / select 1 from cdm_status where status = 'obs_clin' \ No newline at end of file diff --git a/Oracle/obs_gen.sql b/Oracle/obs_gen.sql index 579febb..de668a2 100644 --- a/Oracle/obs_gen.sql +++ b/Oracle/obs_gen.sql @@ -30,8 +30,7 @@ CREATE TABLE obs_gen( ) / update cdm_status -set end_time = sysdate -set records = (select count(*) from obs_gen) +set end_time = sysdate, records = (select count(*) from obs_gen) where task = 'obs_gen' / select 1 from cdm_status where status = 'obs_gen' \ No newline at end of file diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 4fbdabe..3b703d8 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -246,8 +246,7 @@ CREATE OR REPLACE SYNONYM pcornet_enc FOR "&&i2b2_meta_schema".pcornet_enc / update cdm_status -set end_time = sysdate -set records = 0 +set end_time = sysdate, records = 0 where task = 'pcornet_init' / diff --git a/Oracle/pcornet_loader.sql b/Oracle/pcornet_loader.sql index b02c048..513f121 100644 --- a/Oracle/pcornet_loader.sql +++ b/Oracle/pcornet_loader.sql @@ -57,8 +57,7 @@ PCORNetPostProc(); END; / update cdm_status -set end_time = sysdate -set records = 0 +set end_time = sysdate, records = 0 where task = 'pcornet_loader' / select 1 from cdm_status where status = 'pcornet_loader' \ No newline at end of file diff --git a/Oracle/pcornet_trial.sql b/Oracle/pcornet_trial.sql index f406fce..19784cf 100644 --- a/Oracle/pcornet_trial.sql +++ b/Oracle/pcornet_trial.sql @@ -18,8 +18,7 @@ CREATE TABLE pcornet_trial( ) / update cdm_status -set end_time = sysdate -set records = (select count(*) from pcornet_trial) +set end_time = sysdate, records = (select count(*) from pcornet_trial) where task = 'pcornet_trial' / select 1 from cdm_status where status = 'pcornet_trial' \ No newline at end of file diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index bd77c55..095ab0b 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -225,8 +225,7 @@ END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from prescribing) +set end_time = sysdate, records = (select count(*) from prescribing) where task = 'prescribing' / diff --git a/Oracle/pro_cm.sql b/Oracle/pro_cm.sql index f1a69e9..28de5a4 100644 --- a/Oracle/pro_cm.sql +++ b/Oracle/pro_cm.sql @@ -38,8 +38,7 @@ begin end; / update cdm_status -set end_time = sysdate -set records = (select count(*) from pro_cm) +set end_time = sysdate, records = (select count(*) from pro_cm) where task = 'pro_cm' / select 1 from cdm_status where status = 'pro_cm' diff --git a/Oracle/procedures.sql b/Oracle/procedures.sql index 02fd0ef..5c47784 100644 --- a/Oracle/procedures.sql +++ b/Oracle/procedures.sql @@ -63,8 +63,7 @@ PCORNetProcedure(); END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from procedures) +set end_time = sysdate, records = (select count(*) from procedures) where task = 'procedures' / select 1 from cdm_status where status = 'procedures' diff --git a/Oracle/provider.sql b/Oracle/provider.sql index 1a2d8fd..6f6567f 100644 --- a/Oracle/provider.sql +++ b/Oracle/provider.sql @@ -65,8 +65,7 @@ end; / update cdm_status -set end_time = sysdate -set records = (select count(*) from provider) +set end_time = sysdate, records = (select count(*) from provider) where task = 'provider' / diff --git a/Oracle/vital.sql b/Oracle/vital.sql index b7955ab..f90ce3b 100644 --- a/Oracle/vital.sql +++ b/Oracle/vital.sql @@ -138,8 +138,7 @@ PCORNetVital(); END; / update cdm_status -set end_time = sysdate -set records = (select count(*) from vital) +set end_time = sysdate, records = (select count(*) from vital) where task = 'vital' / select 1 from cdm_status where status = 'vital' From a3b72cfb0c22ab05c9abcfc273422e269ec7dfe2 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 11:41:04 -0500 Subject: [PATCH 378/507] Replacing status with task in cdm_status update op. --- Oracle/condition.sql | 2 +- Oracle/death.sql | 2 +- Oracle/death_cause.sql | 2 +- Oracle/demographic.sql | 2 +- Oracle/diagnosis.sql | 2 +- Oracle/dispensing.sql | 2 +- Oracle/encounter.sql | 2 +- Oracle/enrollment.sql | 2 +- Oracle/harvest.sql | 38 +++++++++++++++++++------------------- Oracle/lab_result_cm.sql | 2 +- Oracle/med_admin.sql | 2 +- Oracle/obs_clin.sql | 2 +- Oracle/obs_gen.sql | 2 +- Oracle/pcornet_init.sql | 2 +- Oracle/pcornet_loader.sql | 2 +- Oracle/pcornet_trial.sql | 2 +- Oracle/prescribing.sql | 2 +- Oracle/pro_cm.sql | 2 +- Oracle/procedures.sql | 2 +- Oracle/provider.sql | 2 +- Oracle/vital.sql | 2 +- 21 files changed, 39 insertions(+), 39 deletions(-) diff --git a/Oracle/condition.sql b/Oracle/condition.sql index 6f9a5c7..3bf9114 100644 --- a/Oracle/condition.sql +++ b/Oracle/condition.sql @@ -108,4 +108,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from condition) where task = 'condition' / -select 1 from cdm_status where status = 'condition' \ No newline at end of file +select 1 from cdm_status where task = 'condition' \ No newline at end of file diff --git a/Oracle/death.sql b/Oracle/death.sql index c6a5d6c..595d732 100644 --- a/Oracle/death.sql +++ b/Oracle/death.sql @@ -51,6 +51,6 @@ update cdm_status set end_time = sysdate, records = (select count(*) from death) where task = 'death' / -select 1 from cdm_status where status = 'death' +select 1 from cdm_status where task = 'death' --SELECT count(PATID) from death where rownum = 1 \ No newline at end of file diff --git a/Oracle/death_cause.sql b/Oracle/death_cause.sql index ee61ea6..4f61aeb 100644 --- a/Oracle/death_cause.sql +++ b/Oracle/death_cause.sql @@ -19,5 +19,5 @@ update cdm_status set end_time = sysdate, records = (select count(*) from death_cause) where task = 'death_cause' / -select 1 from cdm_status where status = 'death_cause' +select 1 from cdm_status where task = 'death_cause' --SELECT count(PATID) from death_cause where rownum = 1 \ No newline at end of file diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql index c6352d0..13747bc 100644 --- a/Oracle/demographic.sql +++ b/Oracle/demographic.sql @@ -214,4 +214,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from demographic) where task = 'demographic' / -select 1 from cdm_status where status = 'demographic' \ No newline at end of file +select 1 from cdm_status where task = 'demographic' \ No newline at end of file diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 6a90b76..342dc27 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -251,5 +251,5 @@ update cdm_status set end_time = sysdate, records = (select count(*) from diagnosis) where task = 'diagnosis' / -select 1 from cdm_status where status = 'diagnosis' +select 1 from cdm_status where task = 'diagnosis' --SELECT count(DIAGNOSISID) from diagnosis where rownum = 1 \ No newline at end of file diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index 17a5bd9..992cd61 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -184,5 +184,5 @@ update cdm_status set end_time = sysdate, records = (select count(*) from dispensing) where task = 'dispensing' / -select 1 from cdm_status where status = 'dispensing' +select 1 from cdm_status where task = 'dispensing' --SELECT count(DISPENSINGID) from dispensing where rownum = 1 \ No newline at end of file diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 81a53f2..3f9ea72 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -105,5 +105,5 @@ update cdm_status set end_time = sysdate, records = (select count(*) from encounter) where task = 'encounter' / -select 1 from cdm_status where status = 'encounter' +select 1 from cdm_status where task = 'encounter' --SELECT count(ENCOUNTERID) from encounter where rownum = 1 \ No newline at end of file diff --git a/Oracle/enrollment.sql b/Oracle/enrollment.sql index a65cad5..07d1ddb 100644 --- a/Oracle/enrollment.sql +++ b/Oracle/enrollment.sql @@ -56,5 +56,5 @@ update cdm_status set end_time = sysdate, records = (select count(*) from enrollment) where task = 'enrollment' / -select 1 from cdm_status where status = 'enrollment' +select 1 from cdm_status where task = 'enrollment' --SELECT count(PATID) from enrollment where rownum = 1 \ No newline at end of file diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index 6c42766..c114f6c 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -76,24 +76,24 @@ INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, hl.DEATH_DATE_MGMT, hl.MEDADMIN_START_DATE_MGMT, hl.MEDADMIN_END_DATE_MGMT, hl.OBSCLIN_DATE_MGMT, hl.OBSGEN_DATE_MGMT, - case when (select records from cdm_status where status = 'demographic') > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, - case when (select records from cdm_status where status = 'enrollment') > 0 then current_date else null end REFRESH_ENROLLMENT_DATE, - case when (select records from cdm_status where status = 'encounter') > 0 then current_date else null end REFRESH_ENCOUNTER_DATE, - case when (select records from cdm_status where status = 'diagnosis_finalize') > 0 then current_date else null end REFRESH_DIAGNOSIS_DATE, - case when (select records from cdm_status where status = 'procedures') > 0 then current_date else null end REFRESH_PROCEDURES_DATE, - case when (select records from cdm_status where status = 'vital') > 0 then current_date else null end REFRESH_VITAL_DATE, - case when (select records from cdm_status where status = 'dispensing_finalize') > 0 then current_date else null end REFRESH_DISPENSING_DATE, - case when (select records from cdm_status where status = 'lab_result_cm') > 0 then current_date else null end REFRESH_LAB_RESULT_CM_DATE, - case when (select records from cdm_status where status = 'condition_finalize') > 0 then current_date else null end REFRESH_CONDITION_DATE, - case when (select records from cdm_status where status = 'pro_cm') > 0 then current_date else null end REFRESH_PRO_CM_DATE, - case when (select records from cdm_status where status = 'prescribing') > 0 then current_date else null end REFRESH_PRESCRIBING_DATE, - case when (select records from cdm_status where status = 'pcornet_trial') > 0 then current_date else null end REFRESH_PCORNET_TRIAL_DATE, - case when (select records from cdm_status where status = 'death_finalize') > 0 then current_date else null end REFRESH_DEATH_DATE, - case when (select records from cdm_status where status = 'death_cause') > 0 then current_date else null end REFRESH_DEATH_CAUSE_DATE, - case when (select records from cdm_status where status = 'med_admin') > 0 then current_date else null end REFRESH_MED_ADMIN_DATE, - case when (select records from cdm_status where status = 'obs_clin') > 0 then current_date else null end REFRESH_OBS_CLIN_DATE, - case when (select records from cdm_status where status = 'provider') > 0 then current_date else null end REFRESH_PROVIDER_DATE, - case when (select records from cdm_status where status = 'obs_gen') > 0 then current_date else null end REFRESH_OBS_GEN_DATE + case when (select records from cdm_status where task = 'demographic') > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, + case when (select records from cdm_status where task = 'enrollment') > 0 then current_date else null end REFRESH_ENROLLMENT_DATE, + case when (select records from cdm_status where task = 'encounter') > 0 then current_date else null end REFRESH_ENCOUNTER_DATE, + case when (select records from cdm_status where task = 'diagnosis_finalize') > 0 then current_date else null end REFRESH_DIAGNOSIS_DATE, + case when (select records from cdm_status where task = 'procedures') > 0 then current_date else null end REFRESH_PROCEDURES_DATE, + case when (select records from cdm_status where task = 'vital') > 0 then current_date else null end REFRESH_VITAL_DATE, + case when (select records from cdm_status where task = 'dispensing_finalize') > 0 then current_date else null end REFRESH_DISPENSING_DATE, + case when (select records from cdm_status where task = 'lab_result_cm') > 0 then current_date else null end REFRESH_LAB_RESULT_CM_DATE, + case when (select records from cdm_status where task = 'condition_finalize') > 0 then current_date else null end REFRESH_CONDITION_DATE, + case when (select records from cdm_status where task = 'pro_cm') > 0 then current_date else null end REFRESH_PRO_CM_DATE, + case when (select records from cdm_status where task = 'prescribing') > 0 then current_date else null end REFRESH_PRESCRIBING_DATE, + case when (select records from cdm_status where task = 'pcornet_trial') > 0 then current_date else null end REFRESH_PCORNET_TRIAL_DATE, + case when (select records from cdm_status where task = 'death_finalize') > 0 then current_date else null end REFRESH_DEATH_DATE, + case when (select records from cdm_status where task = 'death_cause') > 0 then current_date else null end REFRESH_DEATH_CAUSE_DATE, + case when (select records from cdm_status where task = 'med_admin') > 0 then current_date else null end REFRESH_MED_ADMIN_DATE, + case when (select records from cdm_status where task = 'obs_clin') > 0 then current_date else null end REFRESH_OBS_CLIN_DATE, + case when (select records from cdm_status where task = 'provider') > 0 then current_date else null end REFRESH_PROVIDER_DATE, + case when (select records from cdm_status where task = 'obs_gen') > 0 then current_date else null end REFRESH_OBS_GEN_DATE from harvest_local hl; @@ -107,4 +107,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from harvest) where task = 'harvest' / -select 1 from cdm_status where status = 'harvest' +select 1 from cdm_status where task = 'harvest' diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index a204d73..0ac4a3f 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -171,4 +171,4 @@ set end_time = sysdate, records = (select count(*) from lab_result_cm) where task = 'lab_result_cm' / -select 1 from cdm_status where status = 'lab_result_cm' \ No newline at end of file +select 1 from cdm_status where task = 'lab_result_cm' \ No newline at end of file diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 3de0021..c2b9391 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -118,4 +118,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from med_admin) where task = 'med_admin' / -select 1 from cdm_status where status = 'med_admin' \ No newline at end of file +select 1 from cdm_status where task = 'med_admin' \ No newline at end of file diff --git a/Oracle/obs_clin.sql b/Oracle/obs_clin.sql index dfe02d1..59c01eb 100644 --- a/Oracle/obs_clin.sql +++ b/Oracle/obs_clin.sql @@ -33,4 +33,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from obs_clin) where task = 'obs_clin' / -select 1 from cdm_status where status = 'obs_clin' \ No newline at end of file +select 1 from cdm_status where task = 'obs_clin' \ No newline at end of file diff --git a/Oracle/obs_gen.sql b/Oracle/obs_gen.sql index de668a2..0c20a33 100644 --- a/Oracle/obs_gen.sql +++ b/Oracle/obs_gen.sql @@ -33,4 +33,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from obs_gen) where task = 'obs_gen' / -select 1 from cdm_status where status = 'obs_gen' \ No newline at end of file +select 1 from cdm_status where task = 'obs_gen' \ No newline at end of file diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 3b703d8..3a86083 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -250,4 +250,4 @@ set end_time = sysdate, records = 0 where task = 'pcornet_init' / -select 1 from cdm_status where status = 'pcornet_init' +select 1 from cdm_status where task = 'pcornet_init' diff --git a/Oracle/pcornet_loader.sql b/Oracle/pcornet_loader.sql index 513f121..a260b42 100644 --- a/Oracle/pcornet_loader.sql +++ b/Oracle/pcornet_loader.sql @@ -60,4 +60,4 @@ update cdm_status set end_time = sysdate, records = 0 where task = 'pcornet_loader' / -select 1 from cdm_status where status = 'pcornet_loader' \ No newline at end of file +select 1 from cdm_status where task = 'pcornet_loader' \ No newline at end of file diff --git a/Oracle/pcornet_trial.sql b/Oracle/pcornet_trial.sql index 19784cf..b5bf5ad 100644 --- a/Oracle/pcornet_trial.sql +++ b/Oracle/pcornet_trial.sql @@ -21,4 +21,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from pcornet_trial) where task = 'pcornet_trial' / -select 1 from cdm_status where status = 'pcornet_trial' \ No newline at end of file +select 1 from cdm_status where task = 'pcornet_trial' \ No newline at end of file diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 095ab0b..43204db 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -229,4 +229,4 @@ set end_time = sysdate, records = (select count(*) from prescribing) where task = 'prescribing' / -select 1 from cdm_status where status = 'prescribing' +select 1 from cdm_status where task = 'prescribing' diff --git a/Oracle/pro_cm.sql b/Oracle/pro_cm.sql index 28de5a4..bb3b5dd 100644 --- a/Oracle/pro_cm.sql +++ b/Oracle/pro_cm.sql @@ -41,4 +41,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from pro_cm) where task = 'pro_cm' / -select 1 from cdm_status where status = 'pro_cm' +select 1 from cdm_status where task = 'pro_cm' diff --git a/Oracle/procedures.sql b/Oracle/procedures.sql index 5c47784..095e2d4 100644 --- a/Oracle/procedures.sql +++ b/Oracle/procedures.sql @@ -66,5 +66,5 @@ update cdm_status set end_time = sysdate, records = (select count(*) from procedures) where task = 'procedures' / -select 1 from cdm_status where status = 'procedures' +select 1 from cdm_status where task = 'procedures' --SELECT count(PATID) from procedures where rownum = 1 \ No newline at end of file diff --git a/Oracle/provider.sql b/Oracle/provider.sql index 6f6567f..b09bd7d 100644 --- a/Oracle/provider.sql +++ b/Oracle/provider.sql @@ -69,4 +69,4 @@ set end_time = sysdate, records = (select count(*) from provider) where task = 'provider' / -select 1 from cdm_status where status = 'provider' +select 1 from cdm_status where task = 'provider' diff --git a/Oracle/vital.sql b/Oracle/vital.sql index f90ce3b..d498f75 100644 --- a/Oracle/vital.sql +++ b/Oracle/vital.sql @@ -141,5 +141,5 @@ update cdm_status set end_time = sysdate, records = (select count(*) from vital) where task = 'vital' / -select 1 from cdm_status where status = 'vital' +select 1 from cdm_status where task = 'vital' --SELECT count(VITALID) from vital where rownum = 1 From 8087dbe1ed68793959868d833363657d712ccc53 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 11:56:08 -0500 Subject: [PATCH 379/507] Debugging STAGEDEV --- etl_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl_tasks.py b/etl_tasks.py index e4a6acd..4c539aa 100644 --- a/etl_tasks.py +++ b/etl_tasks.py @@ -380,7 +380,7 @@ def setTaskEnd(self, rowCount: int): statusTable = Table("cdm_status", MetaData(), Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) db = self._dbtarget().engine - db.execute(statusTable.update, [{'END_TIME': datetime.now(), 'RECORDS': rowCount}]) + db.execute(statusTable.update(), [{'END_TIME': datetime.now(), 'RECORDS': rowCount}]) def setTaskStart(self) -> None: statusTable = Table("cdm_status", MetaData(), Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) From 71e7466d605f9cf3e33bf7860302e31412b96028 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 12:28:07 -0500 Subject: [PATCH 380/507] Debugging STAGEDEV --- Oracle/lab_result_cm.sql | 2 +- i2p_tasks.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 0ac4a3f..afddade 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -64,7 +64,7 @@ select lab_result_cm_seq.nextval LAB_RESULT_CM_ID from blueherondata.observation_fact m join encounter enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num where concept_cd between 'KUH|COMPONENT_ID:' and 'KUH|COMPONENT_ID:~' -and modifier_cd in ('@'); -- exclude analyitics: Labs|Aggregate:Median, ... +and modifier_cd in ('@') -- exclude analyitics: Labs|Aggregate:Median, ... and m.valtype_cd in ('N','T') and (m.nval_num is null or m.nval_num<=9999999) / diff --git a/i2p_tasks.py b/i2p_tasks.py index 8aba0dc..4b38dd4 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -230,7 +230,7 @@ class provider(CDMScriptTask): script = Script.provider def requires(self): - return [encounter()] + return [loadSpecialty(), encounter()] class vital(CDMScriptTask): @@ -267,7 +267,6 @@ def requires(self): class loadSpecialty(LoadCSV): taskName = 'SPECIALTY_MAP' csvname = 'Oracle/specialty.csv' - #csvname = 'Oracle/specialty.csv' def requires(self): return [pcornet_init(), downloadNPI()] @@ -278,7 +277,7 @@ class downloadNPI(CDMStatusTask): expectedRecords = 0 npi_url = 'http://download.cms.gov/nppes/' - dl_path = 'd1/npi/' + dl_path = '/d1/npi/' load_path = 'Oracle/' npi_zip = 'NPPES_Data_Dissemination_040918_041518_Weekly.zip' npi_csv = 'npidata_pfile_20180409-20180415.csv' From e0f8ca9fca80eedddc65d99c342e29fa69f38520 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 12:49:29 -0500 Subject: [PATCH 381/507] Debugging NPI loader --- Oracle/lab_result_cm.sql | 6 +++--- i2p_tasks.py | 9 +++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index afddade..48e56f4 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -93,7 +93,7 @@ inner join select c_fullname, c_basecode from pcornet_lab ) parent -on lab.C_PATH = parent.c_fullname; +on lab.C_PATH = parent.c_fullname / create table lab_result_w_norm as @@ -111,7 +111,7 @@ left join on lab.PATID = norm.patid and lab.RAW_FACILITY_CODE = norm.concept_cd and (lab.LAB_ORDER_DATE - norm.birth_date) > norm.age_lower -and (lab.LAB_ORDER_DATE - norm.birth_date) <= norm.age_upper; +and (lab.LAB_ORDER_DATE - norm.birth_date) <= norm.age_upper / create table lab_result_cm as @@ -155,7 +155,7 @@ select distinct cast(norm.LAB_RESULT_CM_ID as varchar(19)) LAB_RESULT_CM_ID , cast(null as varchar(50)) RAW_UNIT , cast(null as varchar(50)) RAW_ORDER_DEPT , norm.RAW_FACILITY_CODE RAW_FACILITY_CODE -from lab_result_w_norm norm; +from lab_result_w_norm norm / create index lab_result_cm_idx on lab_result_cm (PATID, ENCOUNTERID) diff --git a/i2p_tasks.py b/i2p_tasks.py index 4b38dd4..87237f5 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -300,15 +300,15 @@ def run(self): def fetch(self): r = urllib.request.urlopen(self.npi_url + self.npi_zip) - with open(self.dl_path + self.zip_file, 'wb') as fout: + with open(self.dl_path + self.npi_zip, 'wb') as fout: fout.write(r.read()) - with zipfile.ZipFile(self.dl_path + self.zip_file, 'r') as zip_ref: + with zipfile.ZipFile(self.dl_path + self.npi_zip, 'r') as zip_ref: zip_ref.extractall(self.dl_path) def extract(self): with open(self.dl_path + self.npi_csv, 'r') as fin: - with open(self.dl_path + self.specialty_csv, 'w', newline='') as fout: + with open(self.load_path + self.specialty_csv, 'w', newline='') as fout: reader = csv.DictReader(fin) writer = csv.writer(fout) writer.writerow(['NPI', 'SPECIALTY']) @@ -324,6 +324,3 @@ def extract(self): if (useDefault): writer.writerow([row[self.npi_col], row[self.taxonomy_col + str(1)]]) - def output(self): - return luigi.LocalTarget(self.load_path + self.specialty_csv) - From 33d2c12a5dd2e34947315a167110c8d45e4dbeb0 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 13:57:40 -0500 Subject: [PATCH 382/507] Debugging NPI upload --- i2p_tasks.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index 87237f5..b034f71 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -274,7 +274,6 @@ def requires(self): class downloadNPI(CDMStatusTask): taskName = 'NPI_LOAD' - expectedRecords = 0 npi_url = 'http://download.cms.gov/nppes/' dl_path = '/d1/npi/' @@ -288,9 +287,6 @@ class downloadNPI(CDMStatusTask): npi_col = 'NPI' taxonomy_ct = 15 - def requires(self): - return [] - def run(self): self.setTaskStart() self.fetch() From b793b33c953879247cebae1defff1f90f678a124 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 14:06:21 -0500 Subject: [PATCH 383/507] Debugging NPI upload --- Oracle/pcornet_trial.sql | 4 ++-- i2p_tasks.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Oracle/pcornet_trial.sql b/Oracle/pcornet_trial.sql index b5bf5ad..1bdfed3 100644 --- a/Oracle/pcornet_trial.sql +++ b/Oracle/pcornet_trial.sql @@ -18,7 +18,7 @@ CREATE TABLE pcornet_trial( ) / update cdm_status -set end_time = sysdate, records = (select count(*) from pcornet_trial) +set end_time = sysdate, records = 0 where task = 'pcornet_trial' / -select 1 from cdm_status where task = 'pcornet_trial' \ No newline at end of file +select records + 1 from cdm_status where task = 'pcornet_trial' \ No newline at end of file diff --git a/i2p_tasks.py b/i2p_tasks.py index b034f71..4380189 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -303,14 +303,15 @@ def fetch(self): zip_ref.extractall(self.dl_path) def extract(self): + self.expectedRecords = 0 with open(self.dl_path + self.npi_csv, 'r') as fin: with open(self.load_path + self.specialty_csv, 'w', newline='') as fout: reader = csv.DictReader(fin) writer = csv.writer(fout) writer.writerow(['NPI', 'SPECIALTY']) for row in reader: + self.expectedRecords = self.expectedRecords + 1 useDefault = True - expectedRecords = expectedRecords + 1 for i in range(1, self.taxonomy_ct + 1): if row[self.switch_col + str(i)] == 'Y': useDefault = False From 8e0741905fb78fe4d153d63116e84f634f6f8cbf Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 14:17:39 -0500 Subject: [PATCH 384/507] Debuggin NPI upload --- etl_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl_tasks.py b/etl_tasks.py index 4c539aa..5cb01f2 100644 --- a/etl_tasks.py +++ b/etl_tasks.py @@ -380,7 +380,7 @@ def setTaskEnd(self, rowCount: int): statusTable = Table("cdm_status", MetaData(), Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) db = self._dbtarget().engine - db.execute(statusTable.update(), [{'END_TIME': datetime.now(), 'RECORDS': rowCount}]) + db.execute(statusTable.update().where(statusTable.c.task == self.taskName), [{'END_TIME': datetime.now(), 'RECORDS': rowCount}]) def setTaskStart(self) -> None: statusTable = Table("cdm_status", MetaData(), Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) From 9545c453560e381bf5f073644b620409e543ffb6 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 14:25:15 -0500 Subject: [PATCH 385/507] Debugging NPI upload --- etl_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl_tasks.py b/etl_tasks.py index 5cb01f2..31e8eab 100644 --- a/etl_tasks.py +++ b/etl_tasks.py @@ -380,7 +380,7 @@ def setTaskEnd(self, rowCount: int): statusTable = Table("cdm_status", MetaData(), Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) db = self._dbtarget().engine - db.execute(statusTable.update().where(statusTable.c.task == self.taskName), [{'END_TIME': datetime.now(), 'RECORDS': rowCount}]) + db.execute(statusTable.update().where(statusTable.c.TASK == self.taskName), [{'END_TIME': datetime.now(), 'RECORDS': rowCount}]) def setTaskStart(self) -> None: statusTable = Table("cdm_status", MetaData(), Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) From 0f2877d5c9b090fbff3cb31ad57921c7fde48472 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 14:32:55 -0500 Subject: [PATCH 386/507] Debugging Harvest --- Oracle/harvest.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index c114f6c..36f02ec 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -79,16 +79,16 @@ INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART case when (select records from cdm_status where task = 'demographic') > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, case when (select records from cdm_status where task = 'enrollment') > 0 then current_date else null end REFRESH_ENROLLMENT_DATE, case when (select records from cdm_status where task = 'encounter') > 0 then current_date else null end REFRESH_ENCOUNTER_DATE, - case when (select records from cdm_status where task = 'diagnosis_finalize') > 0 then current_date else null end REFRESH_DIAGNOSIS_DATE, + case when (select records from cdm_status where task = 'diagnosis') > 0 then current_date else null end REFRESH_DIAGNOSIS_DATE, case when (select records from cdm_status where task = 'procedures') > 0 then current_date else null end REFRESH_PROCEDURES_DATE, case when (select records from cdm_status where task = 'vital') > 0 then current_date else null end REFRESH_VITAL_DATE, - case when (select records from cdm_status where task = 'dispensing_finalize') > 0 then current_date else null end REFRESH_DISPENSING_DATE, + case when (select records from cdm_status where task = 'dispensing') > 0 then current_date else null end REFRESH_DISPENSING_DATE, case when (select records from cdm_status where task = 'lab_result_cm') > 0 then current_date else null end REFRESH_LAB_RESULT_CM_DATE, - case when (select records from cdm_status where task = 'condition_finalize') > 0 then current_date else null end REFRESH_CONDITION_DATE, + case when (select records from cdm_status where task = 'condition') > 0 then current_date else null end REFRESH_CONDITION_DATE, case when (select records from cdm_status where task = 'pro_cm') > 0 then current_date else null end REFRESH_PRO_CM_DATE, case when (select records from cdm_status where task = 'prescribing') > 0 then current_date else null end REFRESH_PRESCRIBING_DATE, case when (select records from cdm_status where task = 'pcornet_trial') > 0 then current_date else null end REFRESH_PCORNET_TRIAL_DATE, - case when (select records from cdm_status where task = 'death_finalize') > 0 then current_date else null end REFRESH_DEATH_DATE, + case when (select records from cdm_status where task = 'death') > 0 then current_date else null end REFRESH_DEATH_DATE, case when (select records from cdm_status where task = 'death_cause') > 0 then current_date else null end REFRESH_DEATH_CAUSE_DATE, case when (select records from cdm_status where task = 'med_admin') > 0 then current_date else null end REFRESH_MED_ADMIN_DATE, case when (select records from cdm_status where task = 'obs_clin') > 0 then current_date else null end REFRESH_OBS_CLIN_DATE, @@ -107,4 +107,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from harvest) where task = 'harvest' / -select 1 from cdm_status where task = 'harvest' +select records from cdm_status where task = 'harvest' From 04e36c474f31ae55322ec26f33a5a4953469a304 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 14:36:22 -0500 Subject: [PATCH 387/507] Correcting type in pcornet_trial reference --- Oracle/pcornet_trial.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_trial.sql b/Oracle/pcornet_trial.sql index 1bdfed3..8c6141b 100644 --- a/Oracle/pcornet_trial.sql +++ b/Oracle/pcornet_trial.sql @@ -1,6 +1,6 @@ /** pcornet_trial - create the pcornet_trial table. */ -insert into cdm_status (task, start_time) select 'pcornet_trail', sysdate from dual +insert into cdm_status (task, start_time) select 'pcornet_trial', sysdate from dual / BEGIN PMN_DROPSQL('DROP TABLE pcornet_trial'); From bb9327ff48b4db0d83d70b1da06a38dc7e64f061 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 16:33:53 -0500 Subject: [PATCH 388/507] Add specialty to provider and create curated file directory --- Oracle/demographic.sql | 2 +- Oracle/pcornet_init.sql | 2 +- Oracle/provider.sql | 11 +++++--- {Oracle => curated_data}/harvest_local.csv | 4 +-- {Oracle => curated_data}/labnormal.csv | 0 {Oracle => curated_data}/language.csv | 0 i2p_tasks.py | 33 +++++++++++++--------- 7 files changed, 31 insertions(+), 21 deletions(-) rename {Oracle => curated_data}/harvest_local.csv (99%) rename {Oracle => curated_data}/labnormal.csv (100%) rename {Oracle => curated_data}/language.csv (100%) diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql index 13747bc..972bd61 100644 --- a/Oracle/demographic.sql +++ b/Oracle/demographic.sql @@ -193,7 +193,7 @@ CLOSE getsql; merge into demographic d using ( select NVL(code, 'OT') as code, language_cd, patient_num - from language_map + from language_code right join i2b2patient on case when language_cd is NULL then 'no information' else language_cd end = lower(descriptive_text) ) l diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 3a86083..df1e414 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -121,7 +121,7 @@ END; CREATE TABLE PCORNET_CDM.CDM_STATUS ( - TASK VARCHAR2(50 BYTE) NOT NULL ENABLE, + TASK VARCHAR2(50 BYTE) primary key, START_TIME DATE NULL, END_TIME DATE NULL, RECORDS NUMBER(*,0) NULL diff --git a/Oracle/provider.sql b/Oracle/provider.sql index b09bd7d..8410be4 100644 --- a/Oracle/provider.sql +++ b/Oracle/provider.sql @@ -46,12 +46,15 @@ select cs.prov_id , case when cs.sex = 'U' then 'UN' when cs.sex is null then 'NI' else cs.sex end as sex - , 'OT' + , case when sm.npi is null then 'NI' + else nvl(sm.specialty, 'UN') end as provider_specialty_primary , cs2.npi - , case when npi is not null then 'Y' else 'N' end as provider_npi_flag - , cs.prov_type + , case when cs2.npi is not null then 'Y' else 'N' end as provider_npi_flag + , sp.descriptive_text from clarity.clarity_ser@id cs - join clarity.clarity_ser_2@id cs2 on cs.prov_id = cs2.prov_id; + left join clarity.clarity_ser_2@id cs2 on cs.prov_id = cs2.prov_id + left join provider_specialty_map sm on sm.npi = cs2.npi + left join provider_specialty_code sp on sm.specialty = sp.code; execute immediate 'create index provider_idx on provider (PROVIDERID)'; GATHER_TABLE_STATS('PROVIDER'); diff --git a/Oracle/harvest_local.csv b/curated_data/harvest_local.csv similarity index 99% rename from Oracle/harvest_local.csv rename to curated_data/harvest_local.csv index 8805952..035f41a 100644 --- a/Oracle/harvest_local.csv +++ b/curated_data/harvest_local.csv @@ -1,2 +1,2 @@ -"DATAMART_CLAIMS","DATAMART_EHR","BIRTH_DATE_MGMT","ENR_START_DATE_MGMT","ENR_END_DATE_MGMT","ADMIT_DATE_MGMT","DISCHARGE_DATE_MGMT","PX_DATE_MGMT","RX_ORDER_DATE_MGMT","RX_START_DATE_MGMT","RX_END_DATE_MGMT","DISPENSE_DATE_MGMT","LAB_ORDER_DATE_MGMT","SPECIMEN_DATE_MGMT","RESULT_DATE_MGMT","MEASURE_DATE_MGMT","ONSET_DATE_MGMT","REPORT_DATE_MGMT","RESOLVE_DATE_MGMT","PRO_DATE_MGMT","DEATH_DATE_MGMT","MEDADMIN_START_DATE_MGMT","MEDADMIN_END_DATE_MGMT","OBSCLIN_DATE_MGMT","OBSGEN_DATE_MGMT" -"01","02","03","03","03","03","03","03","03","03","04","03","03","03","03","03","03","03","03","03","03","03","03","03","03" +"DATAMART_CLAIMS","DATAMART_EHR","BIRTH_DATE_MGMT","ENR_START_DATE_MGMT","ENR_END_DATE_MGMT","ADMIT_DATE_MGMT","DISCHARGE_DATE_MGMT","PX_DATE_MGMT","RX_ORDER_DATE_MGMT","RX_START_DATE_MGMT","RX_END_DATE_MGMT","DISPENSE_DATE_MGMT","LAB_ORDER_DATE_MGMT","SPECIMEN_DATE_MGMT","RESULT_DATE_MGMT","MEASURE_DATE_MGMT","ONSET_DATE_MGMT","REPORT_DATE_MGMT","RESOLVE_DATE_MGMT","PRO_DATE_MGMT","DEATH_DATE_MGMT","MEDADMIN_START_DATE_MGMT","MEDADMIN_END_DATE_MGMT","OBSCLIN_DATE_MGMT","OBSGEN_DATE_MGMT" +"01","02","03","03","03","03","03","03","03","03","04","03","03","03","03","03","03","03","03","03","03","03","03","03","03" diff --git a/Oracle/labnormal.csv b/curated_data/labnormal.csv similarity index 100% rename from Oracle/labnormal.csv rename to curated_data/labnormal.csv diff --git a/Oracle/language.csv b/curated_data/language.csv similarity index 100% rename from Oracle/language.csv rename to curated_data/language.csv diff --git a/i2p_tasks.py b/i2p_tasks.py index 4380189..33efd74 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -230,7 +230,7 @@ class provider(CDMScriptTask): script = Script.provider def requires(self): - return [loadSpecialty(), encounter()] + return [loadSpecialtyMap(), loadSpecialtyCode(), encounter()] class vital(CDMScriptTask): @@ -242,7 +242,7 @@ def requires(self): class loadLabNormal(LoadCSV): taskName = 'LABNORMAL' - csvname = 'Oracle/labnormal.csv' + csvname = 'curated_data/labnormal.csv' def requires(self): return [pcornet_init()] @@ -250,37 +250,44 @@ def requires(self): class loadHarvestLocal(LoadCSV): taskName = 'HARVEST_LOCAL' - csvname = 'Oracle/harvest_local.csv' + csvname = 'curated_data/harvest_local.csv' def requires(self): return [pcornet_init()] class loadLanguage(LoadCSV): - taskName = 'LANGUAGE_MAP' - csvname = 'Oracle/language.csv' + taskName = 'LANGUAGE_CODE' + # language.csv is a copy of the CDM spec's patient_pref_language_spoke spreadsheet. + csvname = 'curated_data/language.csv' def requires(self): return [pcornet_init()] -class loadSpecialty(LoadCSV): - taskName = 'SPECIALTY_MAP' - csvname = 'Oracle/specialty.csv' +class loadSpecialtyMap(LoadCSV): + taskName = 'PROVIDER_SPECIALTY_MAP' + # provider_specialty_map.csv is created on demand by the downloadNPI method. + csvname = 'curated_data/provider_specialty_map.csv' def requires(self): return [pcornet_init(), downloadNPI()] +class loadSpecialtyCode(LoadCSV): + taskName = 'PROVIDER_SPECIALTY_CODE' + # provider_specialty_code.csv is a copy of the CDM spec's provider_primary_specialty spreadsheet. + csvname = 'curated_data/provider_specialty_code.csv' + + class downloadNPI(CDMStatusTask): taskName = 'NPI_LOAD' - npi_url = 'http://download.cms.gov/nppes/' dl_path = '/d1/npi/' - load_path = 'Oracle/' - npi_zip = 'NPPES_Data_Dissemination_040918_041518_Weekly.zip' - npi_csv = 'npidata_pfile_20180409-20180415.csv' - specialty_csv = 'specialty.csv' + load_path = 'curated_data/' + npi_zip = 'NPPES_Data_Dissemination_April_2018.zip' + npi_csv = 'npidata_pfile_20050523 - 20180408.csv' + specialty_csv = 'provider_specialty_map.csv' taxonomy_col = 'Healthcare Provider Taxonomy Code_' switch_col = 'Healthcare Provider Primary Taxonomy Switch_' From e50f66d1a651b365a948d97066b79eb13c881e0f Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 16:46:21 -0500 Subject: [PATCH 389/507] Add provider_specialty_code file --- curated_data/provider_specialty_code.csv | 857 +++++++++++++++++++++++ i2p_tasks.py | 2 +- 2 files changed, 858 insertions(+), 1 deletion(-) create mode 100644 curated_data/provider_specialty_code.csv diff --git a/curated_data/provider_specialty_code.csv b/curated_data/provider_specialty_code.csv new file mode 100644 index 0000000..dd3da8a --- /dev/null +++ b/curated_data/provider_specialty_code.csv @@ -0,0 +1,857 @@ +Code,Descriptive_Text,Grouping +101Y00000X,Counselor,Behavioral Health & Social Service Providers +101YA0400X,Counselor Addiction Substance Use Disorder,Behavioral Health & Social Service Providers +101YM0800X,Counselor Mental Health,Behavioral Health & Social Service Providers +101YP1600X,Counselor Pastoral,Behavioral Health & Social Service Providers +101YP2500X,Counselor Professional,Behavioral Health & Social Service Providers +101YS0200X,Counselor School,Behavioral Health & Social Service Providers +102L00000X,Psychoanalyst ,Behavioral Health & Social Service Providers +102X00000X,Poetry Therapist ,Behavioral Health & Social Service Providers +103G00000X,Clinical Neuropsychologist ,Behavioral Health & Social Service Providers +103GC0700X,Clinical Neuropsychologist Clinical,Behavioral Health & Social Service Providers +103K00000X,Behavioral Analyst ,Behavioral Health & Social Service Providers +103T00000X,Psychologist ,Behavioral Health & Social Service Providers +103TA0400X,Psychologist Addiction (Substance Use Disorder),Behavioral Health & Social Service Providers +103TA0700X,Psychologist Adult Development & Aging,Behavioral Health & Social Service Providers +103TB0200X,Psychologist Cognitive & Behavioral,Behavioral Health & Social Service Providers +103TC0700X,Psychologist Clinical,Behavioral Health & Social Service Providers +103TC1900X,Psychologist Counseling,Behavioral Health & Social Service Providers +103TC2200X,Psychologist Clinical Child & Adolescent,Behavioral Health & Social Service Providers +103TE1000X,Psychologist Educational,Behavioral Health & Social Service Providers +103TE1100X,Psychologist Exercise & Sports,Behavioral Health & Social Service Providers +103TF0000X,Psychologist Family,Behavioral Health & Social Service Providers +103TF0200X,Psychologist Forensic,Behavioral Health & Social Service Providers +103TH0004X,Psychologist Health,Behavioral Health & Social Service Providers +103TH0100X,Psychologist Health Service,Behavioral Health & Social Service Providers +103TM1700X,Psychologist Men & Masculinity,Behavioral Health & Social Service Providers +103TM1800X,Psychologist Mental Retardation & Developmental Disabilities,Behavioral Health & Social Service Providers +103TP0016X,Psychologist Prescribing (Medical),Behavioral Health & Social Service Providers +103TP0814X,Psychologist Psychoanalysis,Behavioral Health & Social Service Providers +103TP2700X,Psychologist Psychotherapy,Behavioral Health & Social Service Providers +103TP2701X,Psychologist Group Psychotherapy,Behavioral Health & Social Service Providers +103TR0400X,Psychologist Rehabilitation,Behavioral Health & Social Service Providers +103TS0200X,Psychologist School,Behavioral Health & Social Service Providers +103TW0100X,Psychologist Women,Behavioral Health & Social Service Providers +104100000X,Social Worker ,Behavioral Health & Social Service Providers +1041C0700X,Social Worker Clinical,Behavioral Health & Social Service Providers +1041S0200X,Social Worker School,Behavioral Health & Social Service Providers +106E00000X,Assistant Behavior Analyst ,Behavioral Health & Social Service Providers +106H00000X,Marriage & Family Therapist ,Behavioral Health & Social Service Providers +106S00000X,Behavior Technician ,Behavioral Health & Social Service Providers +111N00000X,Chiropractor ,Chiropractic Providers +111NI0013X,Chiropractor Independent Medical Examiner,Chiropractic Providers +111NI0900X,Chiropractor Internist,Chiropractic Providers +111NN0400X,Chiropractor Neurology,Chiropractic Providers +111NN1001X,Chiropractor Nutrition,Chiropractic Providers +111NP0017X,Chiropractor Pediatric Chiropractor,Chiropractic Providers +111NR0200X,Chiropractor Radiology,Chiropractic Providers +111NR0400X,Chiropractor Rehabilitation,Chiropractic Providers +111NS0005X,Chiropractor Sports Physician,Chiropractic Providers +111NT0100X,Chiropractor Thermography,Chiropractic Providers +111NX0100X,Chiropractor Occupational Health,Chiropractic Providers +111NX0800X,Chiropractor Orthopedic,Chiropractic Providers +122300000X,Dentist ,Dental Providers +1223D0001X,Dentist Dental Public Health,Dental Providers +1223D0004X,Dentist Dentist Anesthesiologist,Dental Providers +1223E0200X,Dentist Endodontics,Dental Providers +1223G0001X,Dentist General Practice,Dental Providers +1223P0106X,Dentist Oral and Maxillofacial Pathology,Dental Providers +1223P0221X,Dentist Pediatric Dentistry,Dental Providers +1223P0300X,Dentist Periodontics,Dental Providers +1223P0700X,Dentist Prosthodontics,Dental Providers +1223S0112X,Dentist Oral and Maxillofacial Surgery,Dental Providers +1223X0008X,Dentist Oral and Maxillofacial Radiology,Dental Providers +1223X0400X,Dentist Orthodontics and Dentofacial Orthopedics,Dental Providers +122400000X,Denturist ,Dental Providers +124Q00000X,Dental Hygienist ,Dental Providers +125J00000X,Dental Therapist ,Dental Providers +125K00000X,Advanced Practice Dental Therapist ,Dental Providers +125Q00000X,Oral Medicinist ,Dental Providers +126800000X,Dental Assistant ,Dental Providers +126900000X,Dental Laboratory Technician ,Dental Providers +132700000X,Dietary Manager ,Dietary & Nutritional Service Providers +133N00000X,Nutritionist ,Dietary & Nutritional Service Providers +133NN1002X,"Nutritionist Nutrition, Education",Dietary & Nutritional Service Providers +133V00000X,"Dietitian, Registered ",Dietary & Nutritional Service Providers +133VN1004X,"Dietitian, Registered Nutrition, Pediatric",Dietary & Nutritional Service Providers +133VN1005X,"Dietitian, Registered Nutrition, Renal",Dietary & Nutritional Service Providers +133VN1006X,"Dietitian, Registered Nutrition, Metabolic",Dietary & Nutritional Service Providers +136A00000X,"Dietetic Technician, Registered ",Dietary & Nutritional Service Providers +146D00000X,Personal Emergency Response Attendant ,Emergency Medical Service Providers +146L00000X,"Emergency Medical Technician, Paramedic ",Emergency Medical Service Providers +146M00000X,"Emergency Medical Technician, Intermediate ",Emergency Medical Service Providers +146N00000X,"Emergency Medical Technician, Basic ",Emergency Medical Service Providers +152W00000X,Optometrist ,Eye and Vision Services Providers +152WC0802X,Optometrist Corneal and Contact Management,Eye and Vision Services Providers +152WL0500X,Optometrist Low Vision Rehabilitation,Eye and Vision Services Providers +152WP0200X,Optometrist Pediatrics,Eye and Vision Services Providers +152WS0006X,Optometrist Sports Vision,Eye and Vision Services Providers +152WV0400X,Optometrist Vision Therapy,Eye and Vision Services Providers +152WX0102X,Optometrist Occupational Vision,Eye and Vision Services Providers +156F00000X,Technician/Technologist ,Eye and Vision Services Providers +156FC0800X,Technician/Technologist Contact Lens,Eye and Vision Services Providers +156FC0801X,Technician/Technologist Contact Lens Fitter,Eye and Vision Services Providers +156FX1100X,Technician/Technologist Ophthalmic,Eye and Vision Services Providers +156FX1101X,Technician/Technologist Ophthalmic Assistant,Eye and Vision Services Providers +156FX1201X,Technician/Technologist Optometric Assistant,Eye and Vision Services Providers +156FX1202X,Technician/Technologist Optometric Technician,Eye and Vision Services Providers +156FX1700X,Technician/Technologist Ocularist,Eye and Vision Services Providers +156FX1800X,Technician/Technologist Optician,Eye and Vision Services Providers +156FX1900X,Technician/Technologist Orthoptist,Eye and Vision Services Providers +163W00000X,Registered Nurse ,Nursing Service Providers +163WA0400X,Registered Nurse Addiction (Substance Use Disorder),Nursing Service Providers +163WA2000X,Registered Nurse Administrator,Nursing Service Providers +163WC0200X,Registered Nurse Critical Care Medicine,Nursing Service Providers +163WC0400X,Registered Nurse Case Management,Nursing Service Providers +163WC1400X,Registered Nurse College Health,Nursing Service Providers +163WC1500X,Registered Nurse Community Health,Nursing Service Providers +163WC1600X,Registered Nurse Continuing Education/Staff Development,Nursing Service Providers +163WC2100X,Registered Nurse Continence Care,Nursing Service Providers +163WC3500X,Registered Nurse Cardiac Rehabilitation,Nursing Service Providers +163WD0400X,Registered Nurse Diabetes Educator,Nursing Service Providers +163WD1100X,"Registered Nurse Dialysis, Peritoneal",Nursing Service Providers +163WE0003X,Registered Nurse Emergency,Nursing Service Providers +163WE0900X,Registered Nurse Enterostomal Therapy,Nursing Service Providers +163WF0300X,Registered Nurse Flight,Nursing Service Providers +163WG0000X,Registered Nurse General Practice,Nursing Service Providers +163WG0100X,Registered Nurse Gastroenterology,Nursing Service Providers +163WG0600X,Registered Nurse Gerontology,Nursing Service Providers +163WH0200X,Registered Nurse Home Health,Nursing Service Providers +163WH0500X,Registered Nurse Hemodialysis,Nursing Service Providers +163WH1000X,Registered Nurse Hospice,Nursing Service Providers +163WI0500X,Registered Nurse Infusion Therapy,Nursing Service Providers +163WI0600X,Registered Nurse Infection Control,Nursing Service Providers +163WL0100X,Registered Nurse Lactation Consultant,Nursing Service Providers +163WM0102X,Registered Nurse Maternal Newborn,Nursing Service Providers +163WM0705X,Registered Nurse Medical-Surgical,Nursing Service Providers +163WM1400X,Registered Nurse Nurse Massage Therapist (NMT),Nursing Service Providers +163WN0002X,Registered Nurse Neonatal Intensive Care,Nursing Service Providers +163WN0003X,"Registered Nurse Neonatal, Low-Risk",Nursing Service Providers +163WN0300X,Registered Nurse Nephrology,Nursing Service Providers +163WN0800X,Registered Nurse Neuroscience,Nursing Service Providers +163WN1003X,Registered Nurse Nutrition Support,Nursing Service Providers +163WP0000X,Registered Nurse Pain Management,Nursing Service Providers +163WP0200X,Registered Nurse Pediatrics,Nursing Service Providers +163WP0218X,Registered Nurse Pediatric Oncology,Nursing Service Providers +163WP0807X,"Registered Nurse Psych/Mental Health, Child & Adolescent",Nursing Service Providers +163WP0808X,Registered Nurse Psych/Mental Health,Nursing Service Providers +163WP0809X,"Registered Nurse Psych/Mental Health, Adult",Nursing Service Providers +163WP1700X,Registered Nurse Perinatal,Nursing Service Providers +163WP2201X,Registered Nurse Ambulatory Care,Nursing Service Providers +163WR0006X,Registered Nurse Registered Nurse First Assistant,Nursing Service Providers +163WR0400X,Registered Nurse Rehabilitation,Nursing Service Providers +163WR1000X,Registered Nurse Reproductive Endocrinology/Infertility,Nursing Service Providers +163WS0121X,Registered Nurse Plastic Surgery,Nursing Service Providers +163WS0200X,Registered Nurse School,Nursing Service Providers +163WU0100X,Registered Nurse Urology,Nursing Service Providers +163WW0000X,Registered Nurse Wound Care,Nursing Service Providers +163WW0101X,"Registered Nurse Women's Health Care, Ambulatory",Nursing Service Providers +163WX0002X,"Registered Nurse Obstetric, High-Risk",Nursing Service Providers +163WX0003X,"Registered Nurse Obstetric, Inpatient",Nursing Service Providers +163WX0106X,Registered Nurse Occupational Health,Nursing Service Providers +163WX0200X,Registered Nurse Oncology,Nursing Service Providers +163WX0601X,Registered Nurse Otorhinolaryngology & Head-Neck,Nursing Service Providers +163WX0800X,Registered Nurse Orthopedic,Nursing Service Providers +163WX1100X,Registered Nurse Ophthalmic,Nursing Service Providers +163WX1500X,Registered Nurse Ostomy Care,Nursing Service Providers +164W00000X,Licensed Practical Nurse ,Nursing Service Providers +164X00000X,Licensed Vocational Nurse ,Nursing Service Providers +167G00000X,Licensed Psychiatric Technician ,Nursing Service Providers +170100000X,"Medical Genetics, Ph.D. Medical Genetics ",Other Service Providers +170300000X,"Genetic Counselor, MS ",Other Service Providers +171000000X,Military Health Care Provider ,Other Service Providers +1710I1002X,Military Health Care Provider Independent Duty Corpsman,Other Service Providers +1710I1003X,Military Health Care Provider Independent Duty Medical Technicians,Other Service Providers +171100000X,Acupuncturist ,Other Service Providers +171M00000X,Case Manager/Care Coordinator ,Other Service Providers +171R00000X,Interpreter ,Other Service Providers +171W00000X,Contractor ,Other Service Providers +171WH0202X,Contractor Home Modifications,Other Service Providers +171WV0202X,Contractor Vehicle Modifications,Other Service Providers +172A00000X,Driver ,Other Service Providers +172M00000X,Mechanotherapist ,Other Service Providers +172P00000X,Naprapath ,Other Service Providers +172V00000X,Community Health Worker ,Other Service Providers +173000000X,Legal Medicine ,Other Service Providers +173C00000X,Reflexologist ,Other Service Providers +173F00000X,"Sleep Specialist, PhD ",Other Service Providers +174200000X,Meals ,Other Service Providers +174400000X,Specialist ,Other Service Providers +1744G0900X,Specialist Graphics Designer,Other Service Providers +1744P3200X,Specialist Prosthetics Case Management,Other Service Providers +1744R1102X,Specialist Research Study,Other Service Providers +1744R1103X,Specialist Research Data Abstracter/Coder,Other Service Providers +174H00000X,Health Educator ,Other Service Providers +174M00000X,Veterinarian ,Other Service Providers +174MM1900X,Veterinarian Medical Research,Other Service Providers +174N00000X,"Lactation Consultant, Non-RN ",Other Service Providers +174V00000X,Clinical Ethicist ,Other Service Providers +175F00000X,Naturopath ,Other Service Providers +175L00000X,Homeopath ,Other Service Providers +175M00000X,"Midwife, Lay ",Other Service Providers +175T00000X,Peer Specialist ,Other Service Providers +176B00000X,Midwife ,Other Service Providers +176P00000X,Funeral Director ,Other Service Providers +177F00000X,Lodging ,Other Service Providers +183500000X,Pharmacist ,Pharmacy Service Providers +1835C0205X,Pharmacist Critical Care,Pharmacy Service Providers +1835G0000X,Pharmacist General Practice,Pharmacy Service Providers +1835G0303X,Pharmacist Geriatric,Pharmacy Service Providers +1835N0905X,Pharmacist Nuclear,Pharmacy Service Providers +1835N1003X,Pharmacist Nutrition Support,Pharmacy Service Providers +1835P0018X,Pharmacist Pharmacist Clinician (PhC)/ Clinical Pharmacy Specialist,Pharmacy Service Providers +1835P0200X,Pharmacist Pediatrics,Pharmacy Service Providers +1835P1200X,Pharmacist Pharmacotherapy,Pharmacy Service Providers +1835P1300X,Pharmacist Psychiatric,Pharmacy Service Providers +1835P2201X,Pharmacist Ambulatory Care,Pharmacy Service Providers +1835X0200X,Pharmacist Oncology,Pharmacy Service Providers +183700000X,Pharmacy Technician ,Pharmacy Service Providers +193200000X,Multi-Specialty ,Group +193400000X,Single Specialty ,Group +202C00000X,Independent Medical Examiner ,Allopathic & Osteopathic Physicians +202K00000X,Phlebology ,Allopathic & Osteopathic Physicians +204C00000X,"Neuromusculoskeletal Medicine, Sports Medicine ",Allopathic & Osteopathic Physicians +204D00000X,Neuromusculoskeletal Medicine & OMM ,Allopathic & Osteopathic Physicians +204E00000X,Oral & Maxillofacial Surgery ,Allopathic & Osteopathic Physicians +204F00000X,Transplant Surgery ,Allopathic & Osteopathic Physicians +204R00000X,Electrodiagnostic Medicine ,Allopathic & Osteopathic Physicians +207K00000X,Allergy & Immunology ,Allopathic & Osteopathic Physicians +207KA0200X,Allergy & Immunology Allergy,Allopathic & Osteopathic Physicians +207KI0005X,Allergy & Immunology Clinical & Laboratory Immunology,Allopathic & Osteopathic Physicians +207L00000X,Anesthesiology ,Allopathic & Osteopathic Physicians +207LA0401X,Anesthesiology Addiction Medicine,Allopathic & Osteopathic Physicians +207LC0200X,Anesthesiology Critical Care Medicine,Allopathic & Osteopathic Physicians +207LH0002X,Anesthesiology Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +207LP2900X,Anesthesiology Pain Medicine,Allopathic & Osteopathic Physicians +207LP3000X,Anesthesiology Pediatric Anesthesiology,Allopathic & Osteopathic Physicians +207N00000X,Dermatology ,Allopathic & Osteopathic Physicians +207ND0101X,Dermatology MOHS-Micrographic Surgery,Allopathic & Osteopathic Physicians +207ND0900X,Dermatology Dermatopathology,Allopathic & Osteopathic Physicians +207NI0002X,Dermatology Clinical & Laboratory Dermatological Immunology,Allopathic & Osteopathic Physicians +207NP0225X,Dermatology Pediatric Dermatology,Allopathic & Osteopathic Physicians +207NS0135X,Dermatology Procedural Dermatology,Allopathic & Osteopathic Physicians +207P00000X,Emergency Medicine ,Allopathic & Osteopathic Physicians +207PE0004X,Emergency Medicine Emergency Medical Services,Allopathic & Osteopathic Physicians +207PE0005X,Emergency Medicine Undersea and Hyperbaric Medicine,Allopathic & Osteopathic Physicians +207PH0002X,Emergency Medicine Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +207PP0204X,Emergency Medicine Pediatric Emergency Medicine,Allopathic & Osteopathic Physicians +207PS0010X,Emergency Medicine Sports Medicine,Allopathic & Osteopathic Physicians +207PT0002X,Emergency Medicine Medical Toxicology,Allopathic & Osteopathic Physicians +207Q00000X,Family Medicine ,Allopathic & Osteopathic Physicians +207QA0000X,Family Medicine Adolescent Medicine,Allopathic & Osteopathic Physicians +207QA0401X,Family Medicine Addiction Medicine,Allopathic & Osteopathic Physicians +207QA0505X,Family Medicine Adult Medicine,Allopathic & Osteopathic Physicians +207QB0002X,Family Medicine Obesity Medicine,Allopathic & Osteopathic Physicians +207QG0300X,Family Medicine Geriatric Medicine,Allopathic & Osteopathic Physicians +207QH0002X,Family Medicine Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +207QS0010X,Family Medicine Sports Medicine,Allopathic & Osteopathic Physicians +207QS1201X,Family Medicine Sleep Medicine,Allopathic & Osteopathic Physicians +207R00000X,Internal Medicine ,Allopathic & Osteopathic Physicians +207RA0000X,Internal Medicine Adolescent Medicine,Allopathic & Osteopathic Physicians +207RA0001X,Internal Medicine Advanced Heart Failure and Transplant Cardiology,Allopathic & Osteopathic Physicians +207RA0201X,Internal Medicine Allergy & Immunology,Allopathic & Osteopathic Physicians +207RA0401X,Internal Medicine Addiction Medicine,Allopathic & Osteopathic Physicians +207RB0002X,Internal Medicine Obesity Medicine,Allopathic & Osteopathic Physicians +207RC0000X,Internal Medicine Cardiovascular Disease,Allopathic & Osteopathic Physicians +207RC0001X,Internal Medicine Clinical Cardiac Electrophysiology,Allopathic & Osteopathic Physicians +207RC0200X,Internal Medicine Critical Care Medicine,Allopathic & Osteopathic Physicians +207RE0101X,"Internal Medicine Endocrinology, Diabetes & Metabolism",Allopathic & Osteopathic Physicians +207RG0100X,Internal Medicine Gastroenterology,Allopathic & Osteopathic Physicians +207RG0300X,Internal Medicine Geriatric Medicine,Allopathic & Osteopathic Physicians +207RH0000X,Internal Medicine Hematology,Allopathic & Osteopathic Physicians +207RH0002X,Internal Medicine Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +207RH0003X,Internal Medicine Hematology & Oncology,Allopathic & Osteopathic Physicians +207RH0005X,Internal Medicine Hypertension Specialist,Allopathic & Osteopathic Physicians +207RI0001X,Internal Medicine Clinical & Laboratory Immunology,Allopathic & Osteopathic Physicians +207RI0008X,Internal Medicine Hepatology,Allopathic & Osteopathic Physicians +207RI0011X,Internal Medicine Interventional Cardiology,Allopathic & Osteopathic Physicians +207RI0200X,Internal Medicine Infectious Disease,Allopathic & Osteopathic Physicians +207RM1200X,Internal Medicine Magnetic Resonance Imaging (MRI),Allopathic & Osteopathic Physicians +207RN0300X,Internal Medicine Nephrology,Allopathic & Osteopathic Physicians +207RP1001X,Internal Medicine Pulmonary Disease,Allopathic & Osteopathic Physicians +207RR0500X,Internal Medicine Rheumatology,Allopathic & Osteopathic Physicians +207RS0010X,Internal Medicine Sports Medicine,Allopathic & Osteopathic Physicians +207RS0012X,Internal Medicine Sleep Medicine,Allopathic & Osteopathic Physicians +207RT0003X,Internal Medicine Transplant Hepatology,Allopathic & Osteopathic Physicians +207RX0202X,Internal Medicine Medical Oncology,Allopathic & Osteopathic Physicians +207SC0300X,Medical Genetics Clinical Cytogenetic,Allopathic & Osteopathic Physicians +207SG0201X,Medical Genetics Clinical Genetics (M.D.),Allopathic & Osteopathic Physicians +207SG0202X,Medical Genetics Clinical Biochemical Genetics,Allopathic & Osteopathic Physicians +207SG0203X,Medical Genetics Clinical Molecular Genetics,Allopathic & Osteopathic Physicians +207SG0205X,Medical Genetics Ph.D. Medical Genetics,Allopathic & Osteopathic Physicians +207SM0001X,Medical Genetics Molecular Genetic Pathology,Allopathic & Osteopathic Physicians +207T00000X,Neurological Surgery ,Allopathic & Osteopathic Physicians +207U00000X,Nuclear Medicine ,Allopathic & Osteopathic Physicians +207UN0901X,Nuclear Medicine Nuclear Cardiology,Allopathic & Osteopathic Physicians +207UN0902X,Nuclear Medicine Nuclear Imaging & Therapy,Allopathic & Osteopathic Physicians +207UN0903X,Nuclear Medicine In Vivo & In Vitro Nuclear Medicine,Allopathic & Osteopathic Physicians +207V00000X,Obstetrics & Gynecology ,Allopathic & Osteopathic Physicians +207VB0002X,Obstetrics & Gynecology Obesity Medicine,Allopathic & Osteopathic Physicians +207VC0200X,Obstetrics & Gynecology Critical Care Medicine,Allopathic & Osteopathic Physicians +207VE0102X,Obstetrics & Gynecology Reproductive Endocrinology,Allopathic & Osteopathic Physicians +207VF0040X,Obstetrics & Gynecology Female Pelvic Medicine and Reconstructive Surgery,Allopathic & Osteopathic Physicians +207VG0400X,Obstetrics & Gynecology Gynecology,Allopathic & Osteopathic Physicians +207VH0002X,Obstetrics & Gynecology Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +207VM0101X,Obstetrics & Gynecology Maternal & Fetal Medicine,Allopathic & Osteopathic Physicians +207VX0000X,Obstetrics & Gynecology Obstetrics,Allopathic & Osteopathic Physicians +207VX0201X,Obstetrics & Gynecology Gynecologic Oncology,Allopathic & Osteopathic Physicians +207W00000X,Ophthalmology ,Allopathic & Osteopathic Physicians +207WX0009X,Ophthalmology Glaucoma Specialist,Allopathic & Osteopathic Physicians +207WX0107X,Ophthalmology Retina Specialist,Allopathic & Osteopathic Physicians +207WX0108X,Ophthalmology Uveitis and Ocular Inflammatory Disease,Allopathic & Osteopathic Physicians +207WX0109X,Ophthalmology Neuro-ophthalmology,Allopathic & Osteopathic Physicians +207WX0110X,Ophthalmology Pediatric Ophthalmology and Strabismus Specialist,Allopathic & Osteopathic Physicians +207WX0200X,Ophthalmology Ophthalmic Plastic and Reconstructive Surgery,Allopathic & Osteopathic Physicians +207X00000X,Orthopaedic Surgery ,Allopathic & Osteopathic Physicians +207XP3100X,Orthopaedic Surgery Pediatric Orthopaedic Surgery,Allopathic & Osteopathic Physicians +207XS0106X,Orthopaedic Surgery Hand Surgery,Allopathic & Osteopathic Physicians +207XS0114X,Orthopaedic Surgery Adult Reconstructive Orthopaedic Surgery,Allopathic & Osteopathic Physicians +207XS0117X,Orthopaedic Surgery Orthopaedic Surgery of the Spine,Allopathic & Osteopathic Physicians +207XX0004X,Orthopaedic Surgery Foot and Ankle Surgery,Allopathic & Osteopathic Physicians +207XX0005X,Orthopaedic Surgery Sports Medicine,Allopathic & Osteopathic Physicians +207XX0801X,Orthopaedic Surgery Orthopaedic Trauma,Allopathic & Osteopathic Physicians +207Y00000X,Otolaryngology ,Allopathic & Osteopathic Physicians +207YP0228X,Otolaryngology Pediatric Otolaryngology,Allopathic & Osteopathic Physicians +207YS0012X,Otolaryngology Sleep Medicine,Allopathic & Osteopathic Physicians +207YS0123X,Otolaryngology Facial Plastic Surgery,Allopathic & Osteopathic Physicians +207YX0007X,Otolaryngology Plastic Surgery within the Head & Neck,Allopathic & Osteopathic Physicians +207YX0602X,Otolaryngology Otolaryngic Allergy,Allopathic & Osteopathic Physicians +207YX0901X,Otolaryngology Otology & Neurotology,Allopathic & Osteopathic Physicians +207YX0905X,Otolaryngology Otolaryngology/Facial Plastic Surgery,Allopathic & Osteopathic Physicians +207ZB0001X,Pathology Blood Banking & Transfusion Medicine,Allopathic & Osteopathic Physicians +207ZC0006X,Pathology Clinical Pathology,Allopathic & Osteopathic Physicians +207ZC0008X,Pathology Clinical Informatics,Allopathic & Osteopathic Physicians +207ZC0500X,Pathology Cytopathology,Allopathic & Osteopathic Physicians +207ZD0900X,Pathology Dermatopathology,Allopathic & Osteopathic Physicians +207ZF0201X,Pathology Forensic Pathology,Allopathic & Osteopathic Physicians +207ZH0000X,Pathology Hematology,Allopathic & Osteopathic Physicians +207ZI0100X,Pathology Immunopathology,Allopathic & Osteopathic Physicians +207ZM0300X,Pathology Medical Microbiology,Allopathic & Osteopathic Physicians +207ZN0500X,Pathology Neuropathology,Allopathic & Osteopathic Physicians +207ZP0007X,Pathology Molecular Genetic Pathology,Allopathic & Osteopathic Physicians +207ZP0101X,Pathology Anatomic Pathology,Allopathic & Osteopathic Physicians +207ZP0102X,Pathology Anatomic Pathology & Clinical Pathology,Allopathic & Osteopathic Physicians +207ZP0104X,Pathology Chemical Pathology,Allopathic & Osteopathic Physicians +207ZP0105X,Pathology Clinical Pathology/Laboratory Medicine,Allopathic & Osteopathic Physicians +207ZP0213X,Pathology Pediatric Pathology,Allopathic & Osteopathic Physicians +208000000X,Pediatrics ,Allopathic & Osteopathic Physicians +2080A0000X,Pediatrics Adolescent Medicine,Allopathic & Osteopathic Physicians +2080B0002X,Pediatrics Obesity Medicine,Allopathic & Osteopathic Physicians +2080C0008X,Pediatrics Child Abuse Pediatrics,Allopathic & Osteopathic Physicians +2080H0002X,Pediatrics Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +2080I0007X,Pediatrics Clinical & Laboratory Immunology,Allopathic & Osteopathic Physicians +2080N0001X,Pediatrics Neonatal-Perinatal Medicine,Allopathic & Osteopathic Physicians +2080P0006X,Pediatrics Developmental – Behavioral Pediatrics,Allopathic & Osteopathic Physicians +2080P0008X,Pediatrics Neurodevelopmental Disabilities,Allopathic & Osteopathic Physicians +2080P0201X,Pediatrics Pediatric Allergy/Immunology,Allopathic & Osteopathic Physicians +2080P0202X,Pediatrics Pediatric Cardiology,Allopathic & Osteopathic Physicians +2080P0203X,Pediatrics Pediatric Critical Care Medicine,Allopathic & Osteopathic Physicians +2080P0204X,Pediatrics Pediatric Emergency Medicine,Allopathic & Osteopathic Physicians +2080P0205X,Pediatrics Pediatric Endocrinology,Allopathic & Osteopathic Physicians +2080P0206X,Pediatrics Pediatric Gastroenterology,Allopathic & Osteopathic Physicians +2080P0207X,Pediatrics Pediatric Hematology-Oncology,Allopathic & Osteopathic Physicians +2080P0208X,Pediatrics Pediatric Infectious Diseases,Allopathic & Osteopathic Physicians +2080P0210X,Pediatrics Pediatric Nephrology,Allopathic & Osteopathic Physicians +2080P0214X,Pediatrics Pediatric Pulmonology,Allopathic & Osteopathic Physicians +2080P0216X,Pediatrics Pediatric Rheumatology,Allopathic & Osteopathic Physicians +2080S0010X,Pediatrics Sports Medicine,Allopathic & Osteopathic Physicians +2080S0012X,Pediatrics Sleep Medicine,Allopathic & Osteopathic Physicians +2080T0002X,Pediatrics Medical Toxicology,Allopathic & Osteopathic Physicians +2080T0004X,Pediatrics Pediatric Transplant Hepatology,Allopathic & Osteopathic Physicians +208100000X,Physical Medicine & Rehabilitation ,Allopathic & Osteopathic Physicians +2081H0002X,Physical Medicine & Rehabilitation Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +2081N0008X,Physical Medicine & Rehabilitation Neuromuscular Medicine,Allopathic & Osteopathic Physicians +2081P0004X,Physical Medicine & Rehabilitation Spinal Cord Injury Medicine,Allopathic & Osteopathic Physicians +2081P0010X,Physical Medicine & Rehabilitation Pediatric Rehabilitation Medicine,Allopathic & Osteopathic Physicians +2081P0301X,Physical Medicine & Rehabilitation Brain Injury Medicine,Allopathic & Osteopathic Physicians +2081P2900X,Physical Medicine & Rehabilitation Pain Medicine,Allopathic & Osteopathic Physicians +2081S0010X,Physical Medicine & Rehabilitation Sports Medicine,Allopathic & Osteopathic Physicians +208200000X,Plastic Surgery ,Allopathic & Osteopathic Physicians +2082S0099X,Plastic Surgery Plastic Surgery Within the Head and Neck,Allopathic & Osteopathic Physicians +2082S0105X,Plastic Surgery Surgery of the Hand,Allopathic & Osteopathic Physicians +2083A0100X,Preventive Medicine Aerospace Medicine,Allopathic & Osteopathic Physicians +2083B0002X,Preventive Medicine Obesity Medicine,Allopathic & Osteopathic Physicians +2083C0008X,Preventive Medicine Clinical Informatics,Allopathic & Osteopathic Physicians +2083P0011X,Preventive Medicine Undersea and Hyperbaric Medicine,Allopathic & Osteopathic Physicians +2083P0500X,Preventive Medicine Preventive Medicine/Occupational Environmental Medicine,Allopathic & Osteopathic Physicians +2083P0901X,Preventive Medicine Public Health & General Preventive Medicine,Allopathic & Osteopathic Physicians +2083S0010X,Preventive Medicine Sports Medicine,Allopathic & Osteopathic Physicians +2083T0002X,Preventive Medicine Medical Toxicology,Allopathic & Osteopathic Physicians +2083X0100X,Preventive Medicine Occupational Medicine,Allopathic & Osteopathic Physicians +2084A0401X,Psychiatry & Neurology Addiction Medicine,Allopathic & Osteopathic Physicians +2084A2900X,Psychiatry & Neurology Neurocritical Care,Allopathic & Osteopathic Physicians +2084B0002X,Psychiatry & Neurology Obesity Medicine,Allopathic & Osteopathic Physicians +2084B0040X,Psychiatry & Neurology Behavioral Neurology & Neuropsychiatry,Allopathic & Osteopathic Physicians +2084D0003X,Psychiatry & Neurology Diagnostic Neuroimaging,Allopathic & Osteopathic Physicians +2084F0202X,Psychiatry & Neurology Forensic Psychiatry,Allopathic & Osteopathic Physicians +2084H0002X,Psychiatry & Neurology Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +2084N0008X,Psychiatry & Neurology Neuromuscular Medicine,Allopathic & Osteopathic Physicians +2084N0400X,Psychiatry & Neurology Neurology,Allopathic & Osteopathic Physicians +2084N0402X,Psychiatry & Neurology Neurology with Special Qualifications in Child Neurology,Allopathic & Osteopathic Physicians +2084N0600X,Psychiatry & Neurology Clinical Neurophysiology,Allopathic & Osteopathic Physicians +2084P0005X,Psychiatry & Neurology Neurodevelopmental Disabilities,Allopathic & Osteopathic Physicians +2084P0015X,Psychiatry & Neurology Psychosomatic Medicine,Allopathic & Osteopathic Physicians +2084P0301X,Psychiatry & Neurology Brain Injury Medicine,Allopathic & Osteopathic Physicians +2084P0800X,Psychiatry & Neurology Psychiatry,Allopathic & Osteopathic Physicians +2084P0802X,Psychiatry & Neurology Addiction Psychiatry,Allopathic & Osteopathic Physicians +2084P0804X,Psychiatry & Neurology Child & Adolescent Psychiatry,Allopathic & Osteopathic Physicians +2084P0805X,Psychiatry & Neurology Geriatric Psychiatry,Allopathic & Osteopathic Physicians +2084P2900X,Psychiatry & Neurology Pain Medicine,Allopathic & Osteopathic Physicians +2084S0010X,Psychiatry & Neurology Sports Medicine,Allopathic & Osteopathic Physicians +2084S0012X,Psychiatry & Neurology Sleep Medicine,Allopathic & Osteopathic Physicians +2084V0102X,Psychiatry & Neurology Vascular Neurology,Allopathic & Osteopathic Physicians +2085B0100X,Radiology Body Imaging,Allopathic & Osteopathic Physicians +2085D0003X,Radiology Diagnostic Neuroimaging,Allopathic & Osteopathic Physicians +2085H0002X,Radiology Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +2085N0700X,Radiology Neuroradiology,Allopathic & Osteopathic Physicians +2085N0904X,Radiology Nuclear Radiology,Allopathic & Osteopathic Physicians +2085P0229X,Radiology Pediatric Radiology,Allopathic & Osteopathic Physicians +2085R0001X,Radiology Radiation Oncology,Allopathic & Osteopathic Physicians +2085R0202X,Radiology Diagnostic Radiology,Allopathic & Osteopathic Physicians +2085R0203X,Radiology Therapeutic Radiology,Allopathic & Osteopathic Physicians +2085R0204X,Radiology Vascular & Interventional Radiology,Allopathic & Osteopathic Physicians +2085R0205X,Radiology Radiological Physics,Allopathic & Osteopathic Physicians +2085U0001X,Radiology Diagnostic Ultrasound,Allopathic & Osteopathic Physicians +208600000X,Surgery ,Allopathic & Osteopathic Physicians +2086H0002X,Surgery Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +2086S0102X,Surgery Surgical Critical Care,Allopathic & Osteopathic Physicians +2086S0105X,Surgery Surgery of the Hand,Allopathic & Osteopathic Physicians +2086S0120X,Surgery Pediatric Surgery,Allopathic & Osteopathic Physicians +2086S0122X,Surgery Plastic and Reconstructive Surgery,Allopathic & Osteopathic Physicians +2086S0127X,Surgery Trauma Surgery,Allopathic & Osteopathic Physicians +2086S0129X,Surgery Vascular Surgery,Allopathic & Osteopathic Physicians +2086X0206X,Surgery Surgical Oncology,Allopathic & Osteopathic Physicians +208800000X,Urology ,Allopathic & Osteopathic Physicians +2088F0040X,Urology Female Pelvic Medicine and Reconstructive Surgery,Allopathic & Osteopathic Physicians +2088P0231X,Urology Pediatric Urology,Allopathic & Osteopathic Physicians +208C00000X,Colon & Rectal Surgery ,Allopathic & Osteopathic Physicians +208D00000X,General Practice ,Allopathic & Osteopathic Physicians +208G00000X,Thoracic Surgery (Cardiothoracic Vascular Surgery) ,Allopathic & Osteopathic Physicians +208M00000X,Hospitalist ,Allopathic & Osteopathic Physicians +208U00000X,Clinical Pharmacology ,Allopathic & Osteopathic Physicians +208VP0000X,Pain Medicine Pain Medicine,Allopathic & Osteopathic Physicians +208VP0014X,Pain Medicine Interventional Pain Medicine,Allopathic & Osteopathic Physicians +209800000X,Legal Medicine ,Allopathic & Osteopathic Physicians +211D00000X,"Assistant, Podiatric ",Podiatric Medicine & Surgery Service Providers +213E00000X,Podiatrist ,Podiatric Medicine & Surgery Service Providers +213EG0000X,Podiatrist General Practice,Podiatric Medicine & Surgery Service Providers +213EP0504X,Podiatrist Public Medicine,Podiatric Medicine & Surgery Service Providers +213EP1101X,Podiatrist Primary Podiatric Medicine,Podiatric Medicine & Surgery Service Providers +213ER0200X,Podiatrist Radiology,Podiatric Medicine & Surgery Service Providers +213ES0000X,Podiatrist Sports Medicine,Podiatric Medicine & Surgery Service Providers +213ES0103X,Podiatrist Foot & Ankle Surgery,Podiatric Medicine & Surgery Service Providers +213ES0131X,Podiatrist Foot Surgery,Podiatric Medicine & Surgery Service Providers +221700000X,Art Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +222Q00000X,Developmental Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +222Z00000X,Orthotist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224900000X,Mastectomy Fitter ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224L00000X,Pedorthist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224P00000X,Prosthetist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224Y00000X,Clinical Exercise Physiologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224Z00000X,Occupational Therapy Assistant ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224ZE0001X,Occupational Therapy Assistant Environmental Modification,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224ZF0002X,"Occupational Therapy Assistant Feeding, Eating & Swallowing","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224ZL0004X,Occupational Therapy Assistant Low Vision,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224ZR0403X,Occupational Therapy Assistant Driving and Community Mobility,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225000000X,Orthotic Fitter ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225100000X,Physical Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251C2600X,Physical Therapist Cardiopulmonary,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251E1200X,Physical Therapist Ergonomics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251E1300X,"Physical Therapist Electrophysiology, Clinical","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251G0304X,Physical Therapist Geriatrics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251H1200X,Physical Therapist Hand,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251H1300X,Physical Therapist Human Factors,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251N0400X,Physical Therapist Neurology,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251P0200X,Physical Therapist Pediatrics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251S0007X,Physical Therapist Sports,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251X0800X,Physical Therapist Orthopedic,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225200000X,Physical Therapy Assistant ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225400000X,Rehabilitation Practitioner ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225500000X,Specialist/Technologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2255A2300X,Specialist/Technologist Athletic Trainer,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2255R0406X,"Specialist/Technologist Rehabilitation, Blind","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225600000X,Dance Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225700000X,Massage Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225800000X,Recreation Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225A00000X,Music Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225B00000X,Pulmonary Function Technologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225C00000X,Rehabilitation Counselor ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225CA2400X,Rehabilitation Counselor Assistive Technology Practitioner,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225CA2500X,Rehabilitation Counselor Assistive Technology Supplier,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225CX0006X,Rehabilitation Counselor Orientation and Mobility Training Provider,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225X00000X,Occupational Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XE0001X,Occupational Therapist Environmental Modification,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XE1200X,Occupational Therapist Ergonomics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XF0002X,"Occupational Therapist Feeding, Eating & Swallowing","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XG0600X,Occupational Therapist Gerontology,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XH1200X,Occupational Therapist Hand,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XH1300X,Occupational Therapist Human Factors,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XL0004X,Occupational Therapist Low Vision,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XM0800X,Occupational Therapist Mental Health,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XN1300X,Occupational Therapist Neurorehabilitation,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XP0019X,Occupational Therapist Physical Rehabilitation,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XP0200X,Occupational Therapist Pediatrics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XR0403X,Occupational Therapist Driving and Community Mobility,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +226000000X,Recreational Therapist Assistant ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +226300000X,Kinesiotherapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +227800000X,"Respiratory Therapist, Certified ","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278C0205X,"Respiratory Therapist, Certified Critical Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278E0002X,"Respiratory Therapist, Certified Emergency Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278E1000X,"Respiratory Therapist, Certified Educational","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278G0305X,"Respiratory Therapist, Certified Geriatric Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278G1100X,"Respiratory Therapist, Certified General Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278H0200X,"Respiratory Therapist, Certified Home Health","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P1004X,"Respiratory Therapist, Certified Pulmonary Diagnostics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P1005X,"Respiratory Therapist, Certified Pulmonary Rehabilitation","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P1006X,"Respiratory Therapist, Certified Pulmonary Function Technologist","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P3800X,"Respiratory Therapist, Certified Palliative/Hospice","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P3900X,"Respiratory Therapist, Certified Neonatal/Pediatrics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P4000X,"Respiratory Therapist, Certified Patient Transport","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278S1500X,"Respiratory Therapist, Certified SNF/Subacute Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +227900000X,"Respiratory Therapist, Registered ","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279C0205X,"Respiratory Therapist, Registered Critical Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279E0002X,"Respiratory Therapist, Registered Emergency Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279E1000X,"Respiratory Therapist, Registered Educational","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279G0305X,"Respiratory Therapist, Registered Geriatric Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279G1100X,"Respiratory Therapist, Registered General Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279H0200X,"Respiratory Therapist, Registered Home Health","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P1004X,"Respiratory Therapist, Registered Pulmonary Diagnostics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P1005X,"Respiratory Therapist, Registered Pulmonary Rehabilitation","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P1006X,"Respiratory Therapist, Registered Pulmonary Function Technologist","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P3800X,"Respiratory Therapist, Registered Palliative/Hospice","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P3900X,"Respiratory Therapist, Registered Neonatal/Pediatrics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P4000X,"Respiratory Therapist, Registered Patient Transport","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279S1500X,"Respiratory Therapist, Registered SNF/Subacute Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +229N00000X,Anaplastologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +231H00000X,Audiologist ,"Speech, Language and Hearing Service Providers" +231HA2400X,Audiologist Assistive Technology Practitioner,"Speech, Language and Hearing Service Providers" +231HA2500X,Audiologist Assistive Technology Supplier,"Speech, Language and Hearing Service Providers" +235500000X,Specialist/Technologist ,"Speech, Language and Hearing Service Providers" +2355A2700X,Specialist/Technologist Audiology Assistant,"Speech, Language and Hearing Service Providers" +2355S0801X,Specialist/Technologist Speech-Language Assistant,"Speech, Language and Hearing Service Providers" +235Z00000X,Speech-Language Pathologist ,"Speech, Language and Hearing Service Providers" +237600000X,Audiologist-Hearing Aid Fitter ,"Speech, Language and Hearing Service Providers" +237700000X,Hearing Instrument Specialist ,"Speech, Language and Hearing Service Providers" +242T00000X,Perfusionist ,"Technologists, Technicians & Other Technical Service Providers" +243U00000X,Radiology Practitioner Assistant ,"Technologists, Technicians & Other Technical Service Providers" +246Q00000X,"Spec/Tech, Pathology ","Technologists, Technicians & Other Technical Service Providers" +246QB0000X,"Spec/Tech, Pathology Blood Banking","Technologists, Technicians & Other Technical Service Providers" +246QC1000X,"Spec/Tech, Pathology Chemistry","Technologists, Technicians & Other Technical Service Providers" +246QC2700X,"Spec/Tech, Pathology Cytotechnology","Technologists, Technicians & Other Technical Service Providers" +246QH0000X,"Spec/Tech, Pathology Hematology","Technologists, Technicians & Other Technical Service Providers" +246QH0401X,"Spec/Tech, Pathology Hemapheresis Practitioner","Technologists, Technicians & Other Technical Service Providers" +246QH0600X,"Spec/Tech, Pathology Histology","Technologists, Technicians & Other Technical Service Providers" +246QI0000X,"Spec/Tech, Pathology Immunology","Technologists, Technicians & Other Technical Service Providers" +246QL0900X,"Spec/Tech, Pathology Laboratory Management","Technologists, Technicians & Other Technical Service Providers" +246QL0901X,"Spec/Tech, Pathology Laboratory Management, Diplomate","Technologists, Technicians & Other Technical Service Providers" +246QM0706X,"Spec/Tech, Pathology Medical Technologist","Technologists, Technicians & Other Technical Service Providers" +246QM0900X,"Spec/Tech, Pathology Microbiology","Technologists, Technicians & Other Technical Service Providers" +246R00000X,"Technician, Pathology ","Technologists, Technicians & Other Technical Service Providers" +246RH0600X,"Technician, Pathology Histology","Technologists, Technicians & Other Technical Service Providers" +246RM2200X,"Technician, Pathology Medical Laboratory","Technologists, Technicians & Other Technical Service Providers" +246RP1900X,"Technician, Pathology Phlebotomy","Technologists, Technicians & Other Technical Service Providers" +246W00000X,"Technician, Cardiology ","Technologists, Technicians & Other Technical Service Providers" +246X00000X,"Spec/Tech, Cardiovascular ","Technologists, Technicians & Other Technical Service Providers" +246XC2901X,"Spec/Tech, Cardiovascular Cardiovascular Invasive Specialist","Technologists, Technicians & Other Technical Service Providers" +246XC2903X,"Spec/Tech, Cardiovascular Vascular Specialist","Technologists, Technicians & Other Technical Service Providers" +246XS1301X,"Spec/Tech, Cardiovascular Sonography","Technologists, Technicians & Other Technical Service Providers" +246Y00000X,"Spec/Tech, Health Info ","Technologists, Technicians & Other Technical Service Providers" +246YC3301X,"Spec/Tech, Health Info Coding Specialist, Hospital Based","Technologists, Technicians & Other Technical Service Providers" +246YC3302X,"Spec/Tech, Health Info Coding Specialist, Physician Office Based","Technologists, Technicians & Other Technical Service Providers" +246YR1600X,"Spec/Tech, Health Info Registered Record Administrator","Technologists, Technicians & Other Technical Service Providers" +246Z00000X,"Specialist/Technologist, Other ","Technologists, Technicians & Other Technical Service Providers" +246ZA2600X,"Specialist/Technologist, Other Art, Medical","Technologists, Technicians & Other Technical Service Providers" +246ZB0301X,"Specialist/Technologist, Other Biomedical Engineering","Technologists, Technicians & Other Technical Service Providers" +246ZB0302X,"Specialist/Technologist, Other Biomedical Photographer","Technologists, Technicians & Other Technical Service Providers" +246ZB0500X,"Specialist/Technologist, Other Biochemist","Technologists, Technicians & Other Technical Service Providers" +246ZB0600X,"Specialist/Technologist, Other Biostatistician","Technologists, Technicians & Other Technical Service Providers" +246ZC0007X,"Specialist/Technologist, Other Surgical Assistant","Technologists, Technicians & Other Technical Service Providers" +246ZE0500X,"Specialist/Technologist, Other EEG","Technologists, Technicians & Other Technical Service Providers" +246ZE0600X,"Specialist/Technologist, Other Electroneurodiagnostic","Technologists, Technicians & Other Technical Service Providers" +246ZG0701X,"Specialist/Technologist, Other Graphics Methods","Technologists, Technicians & Other Technical Service Providers" +246ZG1000X,"Specialist/Technologist, Other Geneticist, Medical (PhD)","Technologists, Technicians & Other Technical Service Providers" +246ZI1000X,"Specialist/Technologist, Other Illustration, Medical","Technologists, Technicians & Other Technical Service Providers" +246ZN0300X,"Specialist/Technologist, Other Nephrology","Technologists, Technicians & Other Technical Service Providers" +246ZS0410X,"Specialist/Technologist, Other Surgical Technologist","Technologists, Technicians & Other Technical Service Providers" +246ZX2200X,"Specialist/Technologist, Other Orthopedic Assistant","Technologists, Technicians & Other Technical Service Providers" +247000000X,"Technician, Health Information ","Technologists, Technicians & Other Technical Service Providers" +2470A2800X,"Technician, Health Information Assistant Record Technician","Technologists, Technicians & Other Technical Service Providers" +247100000X,Radiologic Technologist ,"Technologists, Technicians & Other Technical Service Providers" +2471B0102X,Radiologic Technologist Bone Densitometry,"Technologists, Technicians & Other Technical Service Providers" +2471C1101X,Radiologic Technologist Cardiovascular-Interventional Technology,"Technologists, Technicians & Other Technical Service Providers" +2471C1106X,Radiologic Technologist Cardiac-Interventional Technology,"Technologists, Technicians & Other Technical Service Providers" +2471C3401X,Radiologic Technologist Computed Tomography,"Technologists, Technicians & Other Technical Service Providers" +2471C3402X,Radiologic Technologist Radiography,"Technologists, Technicians & Other Technical Service Providers" +2471M1202X,Radiologic Technologist Magnetic Resonance Imaging,"Technologists, Technicians & Other Technical Service Providers" +2471M2300X,Radiologic Technologist Mammography,"Technologists, Technicians & Other Technical Service Providers" +2471N0900X,Radiologic Technologist Nuclear Medicine Technology,"Technologists, Technicians & Other Technical Service Providers" +2471Q0001X,Radiologic Technologist Quality Management,"Technologists, Technicians & Other Technical Service Providers" +2471R0002X,Radiologic Technologist Radiation Therapy,"Technologists, Technicians & Other Technical Service Providers" +2471S1302X,Radiologic Technologist Sonography,"Technologists, Technicians & Other Technical Service Providers" +2471V0105X,Radiologic Technologist Vascular Sonography,"Technologists, Technicians & Other Technical Service Providers" +2471V0106X,Radiologic Technologist Vascular-Interventional Technology,"Technologists, Technicians & Other Technical Service Providers" +247200000X,"Technician, Other ","Technologists, Technicians & Other Technical Service Providers" +2472B0301X,"Technician, Other Biomedical Engineering","Technologists, Technicians & Other Technical Service Providers" +2472D0500X,"Technician, Other Darkroom","Technologists, Technicians & Other Technical Service Providers" +2472E0500X,"Technician, Other EEG","Technologists, Technicians & Other Technical Service Providers" +2472R0900X,"Technician, Other Renal Dialysis","Technologists, Technicians & Other Technical Service Providers" +2472V0600X,"Technician, Other Veterinary","Technologists, Technicians & Other Technical Service Providers" +247ZC0005X,"Pathology Clinical Laboratory Director, Non-physician","Technologists, Technicians & Other Technical Service Providers" +251300000X,Local Education Agency (LEA) ,Agencies +251B00000X,Case Management ,Agencies +251C00000X,"Day Training, Developmentally Disabled Services ",Agencies +251E00000X,Home Health ,Agencies +251F00000X,Home Infusion ,Agencies +251G00000X,"Hospice Care, Community Based ",Agencies +251J00000X,Nursing Care ,Agencies +251K00000X,Public Health or Welfare ,Agencies +251S00000X,Community/Behavioral Health ,Agencies +251T00000X,PACE Provider Organization ,Agencies +251V00000X,Voluntary or Charitable ,Agencies +251X00000X,Supports Brokerage ,Agencies +252Y00000X,Early Intervention Provider Agency ,Agencies +253J00000X,Foster Care Agency ,Agencies +253Z00000X,In Home Supportive Care ,Agencies +261Q00000X,Clinic/Center ,Ambulatory Health Care Facilities +261QA0005X,Clinic/Center Ambulatory Family Planning Facility,Ambulatory Health Care Facilities +261QA0006X,Clinic/Center Ambulatory Fertility Facility,Ambulatory Health Care Facilities +261QA0600X,Clinic/Center Adult Day Care,Ambulatory Health Care Facilities +261QA0900X,Clinic/Center Amputee,Ambulatory Health Care Facilities +261QA1903X,Clinic/Center Ambulatory Surgical,Ambulatory Health Care Facilities +261QA3000X,Clinic/Center Augmentative Communication,Ambulatory Health Care Facilities +261QB0400X,Clinic/Center Birthing,Ambulatory Health Care Facilities +261QC0050X,Clinic/Center Critical Access Hospital,Ambulatory Health Care Facilities +261QC1500X,Clinic/Center Community Health,Ambulatory Health Care Facilities +261QC1800X,Clinic/Center Corporate Health,Ambulatory Health Care Facilities +261QD0000X,Clinic/Center Dental,Ambulatory Health Care Facilities +261QD1600X,Clinic/Center Developmental Disabilities,Ambulatory Health Care Facilities +261QE0002X,Clinic/Center Emergency Care,Ambulatory Health Care Facilities +261QE0700X,Clinic/Center End-Stage Renal Disease (ESRD) Treatment,Ambulatory Health Care Facilities +261QE0800X,Clinic/Center Endoscopy,Ambulatory Health Care Facilities +261QF0050X,"Clinic/Center Family Planning, Non-Surgical",Ambulatory Health Care Facilities +261QF0400X,Clinic/Center Federally Qualified Health Center (FQHC),Ambulatory Health Care Facilities +261QG0250X,Clinic/Center Genetics,Ambulatory Health Care Facilities +261QH0100X,Clinic/Center Health Service,Ambulatory Health Care Facilities +261QH0700X,Clinic/Center Hearing and Speech,Ambulatory Health Care Facilities +261QI0500X,Clinic/Center Infusion Therapy,Ambulatory Health Care Facilities +261QL0400X,Clinic/Center Lithotripsy,Ambulatory Health Care Facilities +261QM0801X,Clinic/Center Mental Health (Including Community Mental Health Center),Ambulatory Health Care Facilities +261QM0850X,Clinic/Center Adult Mental Health,Ambulatory Health Care Facilities +261QM0855X,Clinic/Center Adolescent and Children Mental Health,Ambulatory Health Care Facilities +261QM1000X,Clinic/Center Migrant Health,Ambulatory Health Care Facilities +261QM1100X,Clinic/Center Military/U.S. Coast Guard Outpatient,Ambulatory Health Care Facilities +261QM1101X,Clinic/Center Military and U.S. Coast Guard Ambulatory Procedure,Ambulatory Health Care Facilities +261QM1102X,Clinic/Center Military Outpatient Operational (Transportable) Component,Ambulatory Health Care Facilities +261QM1103X,Clinic/Center Military Ambulatory Procedure Visits Operational (Transportable),Ambulatory Health Care Facilities +261QM1200X,Clinic/Center Magnetic Resonance Imaging (MRI),Ambulatory Health Care Facilities +261QM1300X,Clinic/Center Multi-Specialty,Ambulatory Health Care Facilities +261QM2500X,Clinic/Center Medical Specialty,Ambulatory Health Care Facilities +261QM2800X,Clinic/Center Methadone Clinic,Ambulatory Health Care Facilities +261QM3000X,Clinic/Center Medically Fragile Intants and Children Day Care,Ambulatory Health Care Facilities +261QP0904X,"Clinic/Center Public Health, Federal",Ambulatory Health Care Facilities +261QP0905X,"Clinic/Center Public Health, State or Local",Ambulatory Health Care Facilities +261QP1100X,Clinic/Center Podiatric,Ambulatory Health Care Facilities +261QP2000X,Clinic/Center Physical Therapy,Ambulatory Health Care Facilities +261QP2300X,Clinic/Center Primary Care,Ambulatory Health Care Facilities +261QP2400X,Clinic/Center Prison Health,Ambulatory Health Care Facilities +261QP3300X,Clinic/Center Pain,Ambulatory Health Care Facilities +261QR0200X,Clinic/Center Radiology,Ambulatory Health Care Facilities +261QR0206X,"Clinic/Center Radiology, Mammography",Ambulatory Health Care Facilities +261QR0207X,"Clinic/Center Radiology, Mobile Mammography",Ambulatory Health Care Facilities +261QR0208X,"Clinic/Center Radiology, Mobile",Ambulatory Health Care Facilities +261QR0400X,Clinic/Center Rehabilitation,Ambulatory Health Care Facilities +261QR0401X,"Clinic/Center Rehabilitation, Comprehensive Outpatient Rehabilitation Facility (CORF)",Ambulatory Health Care Facilities +261QR0404X,"Clinic/Center Rehabilitation, Cardiac Facilities",Ambulatory Health Care Facilities +261QR0405X,"Clinic/Center Rehabilitation, Substance Use Disorder",Ambulatory Health Care Facilities +261QR0800X,Clinic/Center Recovery Care,Ambulatory Health Care Facilities +261QR1100X,Clinic/Center Research,Ambulatory Health Care Facilities +261QR1300X,Clinic/Center Rural Health,Ambulatory Health Care Facilities +261QS0112X,Clinic/Center Oral and Maxillofacial Surgery,Ambulatory Health Care Facilities +261QS0132X,Clinic/Center Ophthalmologic Surgery,Ambulatory Health Care Facilities +261QS1000X,Clinic/Center Student Health,Ambulatory Health Care Facilities +261QS1200X,Clinic/Center Sleep Disorder Diagnostic,Ambulatory Health Care Facilities +261QU0200X,Clinic/Center Urgent Care,Ambulatory Health Care Facilities +261QV0200X,Clinic/Center VA,Ambulatory Health Care Facilities +261QX0100X,Clinic/Center Occupational Medicine,Ambulatory Health Care Facilities +261QX0200X,Clinic/Center Oncology,Ambulatory Health Care Facilities +261QX0203X,"Clinic/Center Oncology, Radiation",Ambulatory Health Care Facilities +273100000X,Epilepsy Unit ,Hospital Units +273R00000X,Psychiatric Unit ,Hospital Units +273Y00000X,Rehabilitation Unit ,Hospital Units +275N00000X,Medicare Defined Swing Bed Unit ,Hospital Units +276400000X,"Rehabilitation, Substance Use Disorder Unit ",Hospital Units +281P00000X,Chronic Disease Hospital ,Hospitals +281PC2000X,Chronic Disease Hospital Children,Hospitals +282E00000X,Long Term Care Hospital ,Hospitals +282J00000X,Religious Nonmedical Health Care Institution ,Hospitals +282N00000X,General Acute Care Hospital ,Hospitals +282NC0060X,General Acute Care Hospital Critical Access,Hospitals +282NC2000X,General Acute Care Hospital Children,Hospitals +282NR1301X,General Acute Care Hospital Rural,Hospitals +282NW0100X,General Acute Care Hospital Women,Hospitals +283Q00000X,Psychiatric Hospital ,Hospitals +283X00000X,Rehabilitation Hospital ,Hospitals +283XC2000X,Rehabilitation Hospital Children,Hospitals +284300000X,Special Hospital ,Hospitals +286500000X,Military Hospital ,Hospitals +2865C1500X,Military Hospital Community Health,Hospitals +2865M2000X,Military Hospital Military General Acute Care Hospital,Hospitals +2865X1600X,Military Hospital Military General Acute Care Hospital. Operational (Transportable),Hospitals +287300000X,Christian Science Sanitorium ,Hospitals +291900000X,Military Clinical Medical Laboratory ,Laboratories +291U00000X,Clinical Medical Laboratory ,Laboratories +292200000X,Dental Laboratory ,Laboratories +293D00000X,Physiological Laboratory ,Laboratories +302F00000X,Exclusive Provider Organization ,Managed Care Organizations +302R00000X,Health Maintenance Organization ,Managed Care Organizations +305R00000X,Preferred Provider Organization ,Managed Care Organizations +305S00000X,Point of Service ,Managed Care Organizations +310400000X,Assisted Living Facility ,Nursing & Custodial Care Facilities +3104A0625X,"Assisted Living Facility Assisted Living, Mental Illness",Nursing & Custodial Care Facilities +3104A0630X,"Assisted Living Facility Assisted Living, Behavioral Disturbances",Nursing & Custodial Care Facilities +310500000X,"Intermediate Care Facility, Mental Illness ",Nursing & Custodial Care Facilities +311500000X,Alzheimer Center (Dementia Center) ,Nursing & Custodial Care Facilities +311Z00000X,Custodial Care Facility ,Nursing & Custodial Care Facilities +311ZA0620X,Custodial Care Facility Adult Care Home,Nursing & Custodial Care Facilities +313M00000X,Nursing Facility/Intermediate Care Facility ,Nursing & Custodial Care Facilities +314000000X,Skilled Nursing Facility ,Nursing & Custodial Care Facilities +3140N1450X,"Skilled Nursing Facility Nursing Care, Pediatric",Nursing & Custodial Care Facilities +315D00000X,"Hospice, Inpatient ",Nursing & Custodial Care Facilities +315P00000X,"Intermediate Care Facility, Mentally Retarded ",Nursing & Custodial Care Facilities +317400000X,Christian Science Facility ,Nursing & Custodial Care Facilities +320600000X,"Residential Treatment Facility, Mental Retardation and/or Developmental Disabilities ",Residential Treatment Facilities +320700000X,"Residential Treatment Facility, Physical Disabilities ",Residential Treatment Facilities +320800000X,"Community Based Residential Treatment Facility, Mental Illness ",Residential Treatment Facilities +320900000X,"Community Based Residential Treatment, Mental Retardation and/or Developmental Disabilities ",Residential Treatment Facilities +322D00000X,"Residential Treatment Facility, Emotionally Disturbed Children ",Residential Treatment Facilities +323P00000X,Psychiatric Residential Treatment Facility ,Residential Treatment Facilities +324500000X,Substance Abuse Rehabilitation Facility ,Residential Treatment Facilities +3245S0500X,"Substance Abuse Rehabilitation Facility Substance Abuse Treatment, Children",Residential Treatment Facilities +331L00000X,Blood Bank ,Suppliers +332000000X,Military/U.S. Coast Guard Pharmacy ,Suppliers +332100000X,Department of Veterans Affairs (VA) Pharmacy ,Suppliers +332800000X,Indian Health Service/Tribal/Urban Indian Health (I/T/U) Pharmacy ,Suppliers +332900000X,Non-Pharmacy Dispensing Site ,Suppliers +332B00000X,Durable Medical Equipment & Medical Supplies ,Suppliers +332BC3200X,Durable Medical Equipment & Medical Supplies Customized Equipment,Suppliers +332BD1200X,Durable Medical Equipment & Medical Supplies Dialysis Equipment & Supplies,Suppliers +332BN1400X,Durable Medical Equipment & Medical Supplies Nursing Facility Supplies,Suppliers +332BP3500X,Durable Medical Equipment & Medical Supplies Parenteral & Enteral Nutrition,Suppliers +332BX2000X,Durable Medical Equipment & Medical Supplies Oxygen Equipment & Supplies,Suppliers +332G00000X,Eye Bank ,Suppliers +332H00000X,"Eyewear Supplier (Equipment, not the service) ",Suppliers +332S00000X,Hearing Aid Equipment ,Suppliers +332U00000X,Home Delivered Meals ,Suppliers +333300000X,Emergency Response System Companies ,Suppliers +333600000X,Pharmacy ,Suppliers +3336C0002X,Pharmacy Clinic Pharmacy,Suppliers +3336C0003X,Pharmacy Community/Retail Pharmacy,Suppliers +3336C0004X,Pharmacy Compounding Pharmacy,Suppliers +3336H0001X,Pharmacy Home Infusion Therapy Pharmacy,Suppliers +3336I0012X,Pharmacy Institutional Pharmacy,Suppliers +3336L0003X,Pharmacy Long Term Care Pharmacy,Suppliers +3336M0002X,Pharmacy Mail Order Pharmacy,Suppliers +3336M0003X,Pharmacy Managed Care Organization Pharmacy,Suppliers +3336N0007X,Pharmacy Nuclear Pharmacy,Suppliers +3336S0011X,Pharmacy Specialty Pharmacy,Suppliers +335E00000X,Prosthetic/Orthotic Supplier ,Suppliers +335G00000X,Medical Foods Supplier ,Suppliers +335U00000X,Organ Procurement Organization ,Suppliers +335V00000X,Portable X-ray and/or Other Portable Diagnostic Imaging Supplier ,Suppliers +341600000X,Ambulance ,Transportation Services +3416A0800X,Ambulance Air Transport,Transportation Services +3416L0300X,Ambulance Land Transport,Transportation Services +3416S0300X,Ambulance Water Transport,Transportation Services +341800000X,Military/U.S. Coast Guard Transport ,Transportation Services +3418M1110X,"Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance, Ground Transport",Transportation Services +3418M1120X,"Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance, Air Transport",Transportation Services +3418M1130X,"Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance, Water Transport",Transportation Services +343800000X,Secured Medical Transport (VAN) ,Transportation Services +343900000X,Non-emergency Medical Transport (VAN) ,Transportation Services +344600000X,Taxi ,Transportation Services +344800000X,Air Carrier ,Transportation Services +347B00000X,Bus ,Transportation Services +347C00000X,Private Vehicle ,Transportation Services +347D00000X,Train ,Transportation Services +347E00000X,Transportation Broker ,Transportation Services +363A00000X,Physician Assistant ,Physician Assistants & Advanced Practice Nursing Providers +363AM0700X,Physician Assistant Medical,Physician Assistants & Advanced Practice Nursing Providers +363AS0400X,Physician Assistant Surgical Technologist,Physician Assistants & Advanced Practice Nursing Providers +363L00000X,Nurse Practitioner ,Physician Assistants & Advanced Practice Nursing Providers +363LA2100X,Nurse Practitioner Acute Care,Physician Assistants & Advanced Practice Nursing Providers +363LA2200X,Nurse Practitioner Adult Health,Physician Assistants & Advanced Practice Nursing Providers +363LC0200X,Nurse Practitioner Critical Care Medicine,Physician Assistants & Advanced Practice Nursing Providers +363LC1500X,Nurse Practitioner Community Health,Physician Assistants & Advanced Practice Nursing Providers +363LF0000X,Nurse Practitioner Family,Physician Assistants & Advanced Practice Nursing Providers +363LG0600X,Nurse Practitioner Gerontology,Physician Assistants & Advanced Practice Nursing Providers +363LN0000X,Nurse Practitioner Neonatal,Physician Assistants & Advanced Practice Nursing Providers +363LN0005X,"Nurse Practitioner Neonatal, Critical Care",Physician Assistants & Advanced Practice Nursing Providers +363LP0200X,Nurse Practitioner Pediatrics,Physician Assistants & Advanced Practice Nursing Providers +363LP0222X,"Nurse Practitioner Pediatrics, Critical Care",Physician Assistants & Advanced Practice Nursing Providers +363LP0808X,Nurse Practitioner Psych/Mental Health,Physician Assistants & Advanced Practice Nursing Providers +363LP1700X,Nurse Practitioner Perinatal,Physician Assistants & Advanced Practice Nursing Providers +363LP2300X,Nurse Practitioner Primary Care,Physician Assistants & Advanced Practice Nursing Providers +363LS0200X,Nurse Practitioner School,Physician Assistants & Advanced Practice Nursing Providers +363LW0102X,Nurse Practitioner Women's Health,Physician Assistants & Advanced Practice Nursing Providers +363LX0001X,Nurse Practitioner Obstetrics & Gynecology,Physician Assistants & Advanced Practice Nursing Providers +363LX0106X,Nurse Practitioner Occupational Health,Physician Assistants & Advanced Practice Nursing Providers +364S00000X,Clinical Nurse Specialist ,Physician Assistants & Advanced Practice Nursing Providers +364SA2100X,Clinical Nurse Specialist Acute Care,Physician Assistants & Advanced Practice Nursing Providers +364SA2200X,Clinical Nurse Specialist Adult Health,Physician Assistants & Advanced Practice Nursing Providers +364SC0200X,Clinical Nurse Specialist Critical Care Medicine,Physician Assistants & Advanced Practice Nursing Providers +364SC1501X,Clinical Nurse Specialist Community Health/Public Health,Physician Assistants & Advanced Practice Nursing Providers +364SC2300X,Clinical Nurse Specialist Chronic Care,Physician Assistants & Advanced Practice Nursing Providers +364SE0003X,Clinical Nurse Specialist Emergency,Physician Assistants & Advanced Practice Nursing Providers +364SE1400X,Clinical Nurse Specialist Ethics,Physician Assistants & Advanced Practice Nursing Providers +364SF0001X,Clinical Nurse Specialist Family Health,Physician Assistants & Advanced Practice Nursing Providers +364SG0600X,Clinical Nurse Specialist Gerontology,Physician Assistants & Advanced Practice Nursing Providers +364SH0200X,Clinical Nurse Specialist Home Health,Physician Assistants & Advanced Practice Nursing Providers +364SH1100X,Clinical Nurse Specialist Holistic,Physician Assistants & Advanced Practice Nursing Providers +364SI0800X,Clinical Nurse Specialist Informatics,Physician Assistants & Advanced Practice Nursing Providers +364SL0600X,Clinical Nurse Specialist Long-Term Care,Physician Assistants & Advanced Practice Nursing Providers +364SM0705X,Clinical Nurse Specialist Medical-Surgical,Physician Assistants & Advanced Practice Nursing Providers +364SN0000X,Clinical Nurse Specialist Neonatal,Physician Assistants & Advanced Practice Nursing Providers +364SN0800X,Clinical Nurse Specialist Neuroscience,Physician Assistants & Advanced Practice Nursing Providers +364SP0200X,Clinical Nurse Specialist Pediatrics,Physician Assistants & Advanced Practice Nursing Providers +364SP0807X,"Clinical Nurse Specialist Psych/Mental Health, Child & Adolescent",Physician Assistants & Advanced Practice Nursing Providers +364SP0808X,Clinical Nurse Specialist Psych/Mental Health,Physician Assistants & Advanced Practice Nursing Providers +364SP0809X,"Clinical Nurse Specialist Psych/Mental Health, Adult",Physician Assistants & Advanced Practice Nursing Providers +364SP0810X,"Clinical Nurse Specialist Psych/Mental Health, Child & Family",Physician Assistants & Advanced Practice Nursing Providers +364SP0811X,"Clinical Nurse Specialist Psych/Mental Health, Chronically Ill",Physician Assistants & Advanced Practice Nursing Providers +364SP0812X,"Clinical Nurse Specialist Psych/Mental Health, Community",Physician Assistants & Advanced Practice Nursing Providers +364SP0813X,"Clinical Nurse Specialist Psych/Mental Health, Geropsychiatric",Physician Assistants & Advanced Practice Nursing Providers +364SP1700X,Clinical Nurse Specialist Perinatal,Physician Assistants & Advanced Practice Nursing Providers +364SP2800X,Clinical Nurse Specialist Perioperative,Physician Assistants & Advanced Practice Nursing Providers +364SR0400X,Clinical Nurse Specialist Rehabilitation,Physician Assistants & Advanced Practice Nursing Providers +364SS0200X,Clinical Nurse Specialist School,Physician Assistants & Advanced Practice Nursing Providers +364ST0500X,Clinical Nurse Specialist Transplantation,Physician Assistants & Advanced Practice Nursing Providers +364SW0102X,Clinical Nurse Specialist Women's Health,Physician Assistants & Advanced Practice Nursing Providers +364SX0106X,Clinical Nurse Specialist Occupational Health,Physician Assistants & Advanced Practice Nursing Providers +364SX0200X,Clinical Nurse Specialist Oncology,Physician Assistants & Advanced Practice Nursing Providers +364SX0204X,"Clinical Nurse Specialist Oncology, Pediatrics",Physician Assistants & Advanced Practice Nursing Providers +367500000X,"Nurse Anesthetist, Certified Registered ",Physician Assistants & Advanced Practice Nursing Providers +367A00000X,Advanced Practice Midwife ,Physician Assistants & Advanced Practice Nursing Providers +367H00000X,Anesthesiologist Assistant ,Physician Assistants & Advanced Practice Nursing Providers +372500000X,Chore Provider ,Nursing Service Related Providers +372600000X,Adult Companion ,Nursing Service Related Providers +373H00000X,Day Training/Habilitation Specialist ,Nursing Service Related Providers +374700000X,Technician ,Nursing Service Related Providers +3747A0650X,Technician Attendant Care Provider,Nursing Service Related Providers +3747P1801X,Technician Personal Care Attendant,Nursing Service Related Providers +374J00000X,Doula ,Nursing Service Related Providers +374K00000X,Religious Nonmedical Practitioner ,Nursing Service Related Providers +374T00000X,Religious Nonmedical Nursing Personnel ,Nursing Service Related Providers +374U00000X,Home Health Aide ,Nursing Service Related Providers +376G00000X,Nursing Home Administrator ,Nursing Service Related Providers +376J00000X,Homemaker ,Nursing Service Related Providers +376K00000X,Nurse's Aide ,Nursing Service Related Providers +385H00000X,Respite Care ,Respite Care Facility +385HR2050X,Respite Care Respite Care Camp,Respite Care Facility +385HR2055X,"Respite Care Respite Care, Mental Illness, Child",Respite Care Facility +385HR2060X,"Respite Care Respite Care, Mental Retardation and/or Developmental Disabilities, Child",Respite Care Facility +385HR2065X,"Respite Care Respite Care, Physical Disabilities, Child",Respite Care Facility +390200000X,Student in an Organized Health Care Education/Training Program ,"Student, Health Care" +405300000X,Prevention Professional ,Other Service Providers +NI,No information, +UN,Unknown, +OT,Other, diff --git a/i2p_tasks.py b/i2p_tasks.py index 33efd74..95e2058 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -281,7 +281,7 @@ class loadSpecialtyCode(LoadCSV): class downloadNPI(CDMStatusTask): - taskName = 'NPI_LOAD' + taskName = 'NPI_DOWNLOAD' npi_url = 'http://download.cms.gov/nppes/' dl_path = '/d1/npi/' load_path = 'curated_data/' From 227947c2c0cc4b9e1a302abb0f86dbebf0170be6 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 17:07:41 -0500 Subject: [PATCH 390/507] Replace ZipFile with subprocess call --- i2p_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index 95e2058..6cf221c 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -11,6 +11,7 @@ import csv import luigi +import subprocess import urllib.request import zipfile @@ -306,8 +307,7 @@ def fetch(self): with open(self.dl_path + self.npi_zip, 'wb') as fout: fout.write(r.read()) - with zipfile.ZipFile(self.dl_path + self.npi_zip, 'r') as zip_ref: - zip_ref.extractall(self.dl_path) + subprocess.call('unzip', '-o', self.dl_path + self.npi_zip) def extract(self): self.expectedRecords = 0 From 605c06a469ff732e3712603a062276551fb3dba2 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 17:19:33 -0500 Subject: [PATCH 391/507] Correcting syntax of subprocess call and scrubbing specialty csv --- curated_data/provider_specialty_code.csv | 446 +++++++++++------------ i2p_tasks.py | 2 +- 2 files changed, 224 insertions(+), 224 deletions(-) diff --git a/curated_data/provider_specialty_code.csv b/curated_data/provider_specialty_code.csv index dd3da8a..b9f9ead 100644 --- a/curated_data/provider_specialty_code.csv +++ b/curated_data/provider_specialty_code.csv @@ -71,16 +71,16 @@ Code,Descriptive_Text,Grouping 126900000X,Dental Laboratory Technician ,Dental Providers 132700000X,Dietary Manager ,Dietary & Nutritional Service Providers 133N00000X,Nutritionist ,Dietary & Nutritional Service Providers -133NN1002X,"Nutritionist Nutrition, Education",Dietary & Nutritional Service Providers -133V00000X,"Dietitian, Registered ",Dietary & Nutritional Service Providers -133VN1004X,"Dietitian, Registered Nutrition, Pediatric",Dietary & Nutritional Service Providers -133VN1005X,"Dietitian, Registered Nutrition, Renal",Dietary & Nutritional Service Providers -133VN1006X,"Dietitian, Registered Nutrition, Metabolic",Dietary & Nutritional Service Providers -136A00000X,"Dietetic Technician, Registered ",Dietary & Nutritional Service Providers +133NN1002X,Nutritionist Nutrition Education,Dietary & Nutritional Service Providers +133V00000X,Dietitian Registered ,Dietary & Nutritional Service Providers +133VN1004X,Dietitian Registered Nutrition Pediatric,Dietary & Nutritional Service Providers +133VN1005X,Dietitian Registered Nutrition Renal,Dietary & Nutritional Service Providers +133VN1006X,Dietitian Registered Nutrition Metabolic,Dietary & Nutritional Service Providers +136A00000X,Dietetic Technician Registered ,Dietary & Nutritional Service Providers 146D00000X,Personal Emergency Response Attendant ,Emergency Medical Service Providers -146L00000X,"Emergency Medical Technician, Paramedic ",Emergency Medical Service Providers -146M00000X,"Emergency Medical Technician, Intermediate ",Emergency Medical Service Providers -146N00000X,"Emergency Medical Technician, Basic ",Emergency Medical Service Providers +146L00000X,Emergency Medical Technician Paramedic ,Emergency Medical Service Providers +146M00000X,Emergency Medical Technician Intermediate ,Emergency Medical Service Providers +146N00000X,Emergency Medical Technician Basic ,Emergency Medical Service Providers 152W00000X,Optometrist ,Eye and Vision Services Providers 152WC0802X,Optometrist Corneal and Contact Management,Eye and Vision Services Providers 152WL0500X,Optometrist Low Vision Rehabilitation,Eye and Vision Services Providers @@ -109,7 +109,7 @@ Code,Descriptive_Text,Grouping 163WC2100X,Registered Nurse Continence Care,Nursing Service Providers 163WC3500X,Registered Nurse Cardiac Rehabilitation,Nursing Service Providers 163WD0400X,Registered Nurse Diabetes Educator,Nursing Service Providers -163WD1100X,"Registered Nurse Dialysis, Peritoneal",Nursing Service Providers +163WD1100X,Registered Nurse Dialysis Peritoneal,Nursing Service Providers 163WE0003X,Registered Nurse Emergency,Nursing Service Providers 163WE0900X,Registered Nurse Enterostomal Therapy,Nursing Service Providers 163WF0300X,Registered Nurse Flight,Nursing Service Providers @@ -126,16 +126,16 @@ Code,Descriptive_Text,Grouping 163WM0705X,Registered Nurse Medical-Surgical,Nursing Service Providers 163WM1400X,Registered Nurse Nurse Massage Therapist (NMT),Nursing Service Providers 163WN0002X,Registered Nurse Neonatal Intensive Care,Nursing Service Providers -163WN0003X,"Registered Nurse Neonatal, Low-Risk",Nursing Service Providers +163WN0003X,Registered Nurse Neonatal Low-Risk,Nursing Service Providers 163WN0300X,Registered Nurse Nephrology,Nursing Service Providers 163WN0800X,Registered Nurse Neuroscience,Nursing Service Providers 163WN1003X,Registered Nurse Nutrition Support,Nursing Service Providers 163WP0000X,Registered Nurse Pain Management,Nursing Service Providers 163WP0200X,Registered Nurse Pediatrics,Nursing Service Providers 163WP0218X,Registered Nurse Pediatric Oncology,Nursing Service Providers -163WP0807X,"Registered Nurse Psych/Mental Health, Child & Adolescent",Nursing Service Providers +163WP0807X,Registered Nurse Psych/Mental Health Child & Adolescent,Nursing Service Providers 163WP0808X,Registered Nurse Psych/Mental Health,Nursing Service Providers -163WP0809X,"Registered Nurse Psych/Mental Health, Adult",Nursing Service Providers +163WP0809X,Registered Nurse Psych/Mental Health Adult,Nursing Service Providers 163WP1700X,Registered Nurse Perinatal,Nursing Service Providers 163WP2201X,Registered Nurse Ambulatory Care,Nursing Service Providers 163WR0006X,Registered Nurse Registered Nurse First Assistant,Nursing Service Providers @@ -145,9 +145,9 @@ Code,Descriptive_Text,Grouping 163WS0200X,Registered Nurse School,Nursing Service Providers 163WU0100X,Registered Nurse Urology,Nursing Service Providers 163WW0000X,Registered Nurse Wound Care,Nursing Service Providers -163WW0101X,"Registered Nurse Women's Health Care, Ambulatory",Nursing Service Providers -163WX0002X,"Registered Nurse Obstetric, High-Risk",Nursing Service Providers -163WX0003X,"Registered Nurse Obstetric, Inpatient",Nursing Service Providers +163WW0101X,Registered Nurse Women's Health Care Ambulatory,Nursing Service Providers +163WX0002X,Registered Nurse Obstetric High-Risk,Nursing Service Providers +163WX0003X,Registered Nurse Obstetric Inpatient,Nursing Service Providers 163WX0106X,Registered Nurse Occupational Health,Nursing Service Providers 163WX0200X,Registered Nurse Oncology,Nursing Service Providers 163WX0601X,Registered Nurse Otorhinolaryngology & Head-Neck,Nursing Service Providers @@ -157,8 +157,8 @@ Code,Descriptive_Text,Grouping 164W00000X,Licensed Practical Nurse ,Nursing Service Providers 164X00000X,Licensed Vocational Nurse ,Nursing Service Providers 167G00000X,Licensed Psychiatric Technician ,Nursing Service Providers -170100000X,"Medical Genetics, Ph.D. Medical Genetics ",Other Service Providers -170300000X,"Genetic Counselor, MS ",Other Service Providers +170100000X,Medical Genetics Ph.D. Medical Genetics ,Other Service Providers +170300000X,Genetic Counselor MS ,Other Service Providers 171000000X,Military Health Care Provider ,Other Service Providers 1710I1002X,Military Health Care Provider Independent Duty Corpsman,Other Service Providers 1710I1003X,Military Health Care Provider Independent Duty Medical Technicians,Other Service Providers @@ -174,7 +174,7 @@ Code,Descriptive_Text,Grouping 172V00000X,Community Health Worker ,Other Service Providers 173000000X,Legal Medicine ,Other Service Providers 173C00000X,Reflexologist ,Other Service Providers -173F00000X,"Sleep Specialist, PhD ",Other Service Providers +173F00000X,Sleep Specialist PhD ,Other Service Providers 174200000X,Meals ,Other Service Providers 174400000X,Specialist ,Other Service Providers 1744G0900X,Specialist Graphics Designer,Other Service Providers @@ -184,11 +184,11 @@ Code,Descriptive_Text,Grouping 174H00000X,Health Educator ,Other Service Providers 174M00000X,Veterinarian ,Other Service Providers 174MM1900X,Veterinarian Medical Research,Other Service Providers -174N00000X,"Lactation Consultant, Non-RN ",Other Service Providers +174N00000X,Lactation Consultant Non-RN ,Other Service Providers 174V00000X,Clinical Ethicist ,Other Service Providers 175F00000X,Naturopath ,Other Service Providers 175L00000X,Homeopath ,Other Service Providers -175M00000X,"Midwife, Lay ",Other Service Providers +175M00000X,Midwife Lay ,Other Service Providers 175T00000X,Peer Specialist ,Other Service Providers 176B00000X,Midwife ,Other Service Providers 176P00000X,Funeral Director ,Other Service Providers @@ -210,7 +210,7 @@ Code,Descriptive_Text,Grouping 193400000X,Single Specialty ,Group 202C00000X,Independent Medical Examiner ,Allopathic & Osteopathic Physicians 202K00000X,Phlebology ,Allopathic & Osteopathic Physicians -204C00000X,"Neuromusculoskeletal Medicine, Sports Medicine ",Allopathic & Osteopathic Physicians +204C00000X,Neuromusculoskeletal Medicine Sports Medicine ,Allopathic & Osteopathic Physicians 204D00000X,Neuromusculoskeletal Medicine & OMM ,Allopathic & Osteopathic Physicians 204E00000X,Oral & Maxillofacial Surgery ,Allopathic & Osteopathic Physicians 204F00000X,Transplant Surgery ,Allopathic & Osteopathic Physicians @@ -255,7 +255,7 @@ Code,Descriptive_Text,Grouping 207RC0000X,Internal Medicine Cardiovascular Disease,Allopathic & Osteopathic Physicians 207RC0001X,Internal Medicine Clinical Cardiac Electrophysiology,Allopathic & Osteopathic Physicians 207RC0200X,Internal Medicine Critical Care Medicine,Allopathic & Osteopathic Physicians -207RE0101X,"Internal Medicine Endocrinology, Diabetes & Metabolism",Allopathic & Osteopathic Physicians +207RE0101X,Internal Medicine Endocrinology Diabetes & Metabolism,Allopathic & Osteopathic Physicians 207RG0100X,Internal Medicine Gastroenterology,Allopathic & Osteopathic Physicians 207RG0300X,Internal Medicine Geriatric Medicine,Allopathic & Osteopathic Physicians 207RH0000X,Internal Medicine Hematology,Allopathic & Osteopathic Physicians @@ -432,7 +432,7 @@ Code,Descriptive_Text,Grouping 208VP0000X,Pain Medicine Pain Medicine,Allopathic & Osteopathic Physicians 208VP0014X,Pain Medicine Interventional Pain Medicine,Allopathic & Osteopathic Physicians 209800000X,Legal Medicine ,Allopathic & Osteopathic Physicians -211D00000X,"Assistant, Podiatric ",Podiatric Medicine & Surgery Service Providers +211D00000X,Assistant Podiatric ,Podiatric Medicine & Surgery Service Providers 213E00000X,Podiatrist ,Podiatric Medicine & Surgery Service Providers 213EG0000X,Podiatrist General Practice,Podiatric Medicine & Surgery Service Providers 213EP0504X,Podiatrist Public Medicine,Podiatric Medicine & Surgery Service Providers @@ -441,168 +441,168 @@ Code,Descriptive_Text,Grouping 213ES0000X,Podiatrist Sports Medicine,Podiatric Medicine & Surgery Service Providers 213ES0103X,Podiatrist Foot & Ankle Surgery,Podiatric Medicine & Surgery Service Providers 213ES0131X,Podiatrist Foot Surgery,Podiatric Medicine & Surgery Service Providers -221700000X,Art Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -222Q00000X,Developmental Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -222Z00000X,Orthotist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -224900000X,Mastectomy Fitter ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -224L00000X,Pedorthist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -224P00000X,Prosthetist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -224Y00000X,Clinical Exercise Physiologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -224Z00000X,Occupational Therapy Assistant ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -224ZE0001X,Occupational Therapy Assistant Environmental Modification,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -224ZF0002X,"Occupational Therapy Assistant Feeding, Eating & Swallowing","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -224ZL0004X,Occupational Therapy Assistant Low Vision,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -224ZR0403X,Occupational Therapy Assistant Driving and Community Mobility,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225000000X,Orthotic Fitter ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225100000X,Physical Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2251C2600X,Physical Therapist Cardiopulmonary,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2251E1200X,Physical Therapist Ergonomics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2251E1300X,"Physical Therapist Electrophysiology, Clinical","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2251G0304X,Physical Therapist Geriatrics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2251H1200X,Physical Therapist Hand,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2251H1300X,Physical Therapist Human Factors,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2251N0400X,Physical Therapist Neurology,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2251P0200X,Physical Therapist Pediatrics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2251S0007X,Physical Therapist Sports,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2251X0800X,Physical Therapist Orthopedic,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225200000X,Physical Therapy Assistant ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225400000X,Rehabilitation Practitioner ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225500000X,Specialist/Technologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2255A2300X,Specialist/Technologist Athletic Trainer,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2255R0406X,"Specialist/Technologist Rehabilitation, Blind","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225600000X,Dance Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225700000X,Massage Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225800000X,Recreation Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225A00000X,Music Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225B00000X,Pulmonary Function Technologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225C00000X,Rehabilitation Counselor ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225CA2400X,Rehabilitation Counselor Assistive Technology Practitioner,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225CA2500X,Rehabilitation Counselor Assistive Technology Supplier,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225CX0006X,Rehabilitation Counselor Orientation and Mobility Training Provider,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225X00000X,Occupational Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225XE0001X,Occupational Therapist Environmental Modification,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225XE1200X,Occupational Therapist Ergonomics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225XF0002X,"Occupational Therapist Feeding, Eating & Swallowing","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225XG0600X,Occupational Therapist Gerontology,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225XH1200X,Occupational Therapist Hand,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225XH1300X,Occupational Therapist Human Factors,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225XL0004X,Occupational Therapist Low Vision,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225XM0800X,Occupational Therapist Mental Health,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225XN1300X,Occupational Therapist Neurorehabilitation,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225XP0019X,Occupational Therapist Physical Rehabilitation,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225XP0200X,Occupational Therapist Pediatrics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -225XR0403X,Occupational Therapist Driving and Community Mobility,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -226000000X,Recreational Therapist Assistant ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -226300000X,Kinesiotherapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -227800000X,"Respiratory Therapist, Certified ","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278C0205X,"Respiratory Therapist, Certified Critical Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278E0002X,"Respiratory Therapist, Certified Emergency Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278E1000X,"Respiratory Therapist, Certified Educational","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278G0305X,"Respiratory Therapist, Certified Geriatric Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278G1100X,"Respiratory Therapist, Certified General Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278H0200X,"Respiratory Therapist, Certified Home Health","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278P1004X,"Respiratory Therapist, Certified Pulmonary Diagnostics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278P1005X,"Respiratory Therapist, Certified Pulmonary Rehabilitation","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278P1006X,"Respiratory Therapist, Certified Pulmonary Function Technologist","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278P3800X,"Respiratory Therapist, Certified Palliative/Hospice","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278P3900X,"Respiratory Therapist, Certified Neonatal/Pediatrics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278P4000X,"Respiratory Therapist, Certified Patient Transport","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2278S1500X,"Respiratory Therapist, Certified SNF/Subacute Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -227900000X,"Respiratory Therapist, Registered ","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279C0205X,"Respiratory Therapist, Registered Critical Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279E0002X,"Respiratory Therapist, Registered Emergency Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279E1000X,"Respiratory Therapist, Registered Educational","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279G0305X,"Respiratory Therapist, Registered Geriatric Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279G1100X,"Respiratory Therapist, Registered General Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279H0200X,"Respiratory Therapist, Registered Home Health","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279P1004X,"Respiratory Therapist, Registered Pulmonary Diagnostics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279P1005X,"Respiratory Therapist, Registered Pulmonary Rehabilitation","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279P1006X,"Respiratory Therapist, Registered Pulmonary Function Technologist","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279P3800X,"Respiratory Therapist, Registered Palliative/Hospice","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279P3900X,"Respiratory Therapist, Registered Neonatal/Pediatrics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279P4000X,"Respiratory Therapist, Registered Patient Transport","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -2279S1500X,"Respiratory Therapist, Registered SNF/Subacute Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -229N00000X,Anaplastologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" -231H00000X,Audiologist ,"Speech, Language and Hearing Service Providers" -231HA2400X,Audiologist Assistive Technology Practitioner,"Speech, Language and Hearing Service Providers" -231HA2500X,Audiologist Assistive Technology Supplier,"Speech, Language and Hearing Service Providers" -235500000X,Specialist/Technologist ,"Speech, Language and Hearing Service Providers" -2355A2700X,Specialist/Technologist Audiology Assistant,"Speech, Language and Hearing Service Providers" -2355S0801X,Specialist/Technologist Speech-Language Assistant,"Speech, Language and Hearing Service Providers" -235Z00000X,Speech-Language Pathologist ,"Speech, Language and Hearing Service Providers" -237600000X,Audiologist-Hearing Aid Fitter ,"Speech, Language and Hearing Service Providers" -237700000X,Hearing Instrument Specialist ,"Speech, Language and Hearing Service Providers" -242T00000X,Perfusionist ,"Technologists, Technicians & Other Technical Service Providers" -243U00000X,Radiology Practitioner Assistant ,"Technologists, Technicians & Other Technical Service Providers" -246Q00000X,"Spec/Tech, Pathology ","Technologists, Technicians & Other Technical Service Providers" -246QB0000X,"Spec/Tech, Pathology Blood Banking","Technologists, Technicians & Other Technical Service Providers" -246QC1000X,"Spec/Tech, Pathology Chemistry","Technologists, Technicians & Other Technical Service Providers" -246QC2700X,"Spec/Tech, Pathology Cytotechnology","Technologists, Technicians & Other Technical Service Providers" -246QH0000X,"Spec/Tech, Pathology Hematology","Technologists, Technicians & Other Technical Service Providers" -246QH0401X,"Spec/Tech, Pathology Hemapheresis Practitioner","Technologists, Technicians & Other Technical Service Providers" -246QH0600X,"Spec/Tech, Pathology Histology","Technologists, Technicians & Other Technical Service Providers" -246QI0000X,"Spec/Tech, Pathology Immunology","Technologists, Technicians & Other Technical Service Providers" -246QL0900X,"Spec/Tech, Pathology Laboratory Management","Technologists, Technicians & Other Technical Service Providers" -246QL0901X,"Spec/Tech, Pathology Laboratory Management, Diplomate","Technologists, Technicians & Other Technical Service Providers" -246QM0706X,"Spec/Tech, Pathology Medical Technologist","Technologists, Technicians & Other Technical Service Providers" -246QM0900X,"Spec/Tech, Pathology Microbiology","Technologists, Technicians & Other Technical Service Providers" -246R00000X,"Technician, Pathology ","Technologists, Technicians & Other Technical Service Providers" -246RH0600X,"Technician, Pathology Histology","Technologists, Technicians & Other Technical Service Providers" -246RM2200X,"Technician, Pathology Medical Laboratory","Technologists, Technicians & Other Technical Service Providers" -246RP1900X,"Technician, Pathology Phlebotomy","Technologists, Technicians & Other Technical Service Providers" -246W00000X,"Technician, Cardiology ","Technologists, Technicians & Other Technical Service Providers" -246X00000X,"Spec/Tech, Cardiovascular ","Technologists, Technicians & Other Technical Service Providers" -246XC2901X,"Spec/Tech, Cardiovascular Cardiovascular Invasive Specialist","Technologists, Technicians & Other Technical Service Providers" -246XC2903X,"Spec/Tech, Cardiovascular Vascular Specialist","Technologists, Technicians & Other Technical Service Providers" -246XS1301X,"Spec/Tech, Cardiovascular Sonography","Technologists, Technicians & Other Technical Service Providers" -246Y00000X,"Spec/Tech, Health Info ","Technologists, Technicians & Other Technical Service Providers" -246YC3301X,"Spec/Tech, Health Info Coding Specialist, Hospital Based","Technologists, Technicians & Other Technical Service Providers" -246YC3302X,"Spec/Tech, Health Info Coding Specialist, Physician Office Based","Technologists, Technicians & Other Technical Service Providers" -246YR1600X,"Spec/Tech, Health Info Registered Record Administrator","Technologists, Technicians & Other Technical Service Providers" -246Z00000X,"Specialist/Technologist, Other ","Technologists, Technicians & Other Technical Service Providers" -246ZA2600X,"Specialist/Technologist, Other Art, Medical","Technologists, Technicians & Other Technical Service Providers" -246ZB0301X,"Specialist/Technologist, Other Biomedical Engineering","Technologists, Technicians & Other Technical Service Providers" -246ZB0302X,"Specialist/Technologist, Other Biomedical Photographer","Technologists, Technicians & Other Technical Service Providers" -246ZB0500X,"Specialist/Technologist, Other Biochemist","Technologists, Technicians & Other Technical Service Providers" -246ZB0600X,"Specialist/Technologist, Other Biostatistician","Technologists, Technicians & Other Technical Service Providers" -246ZC0007X,"Specialist/Technologist, Other Surgical Assistant","Technologists, Technicians & Other Technical Service Providers" -246ZE0500X,"Specialist/Technologist, Other EEG","Technologists, Technicians & Other Technical Service Providers" -246ZE0600X,"Specialist/Technologist, Other Electroneurodiagnostic","Technologists, Technicians & Other Technical Service Providers" -246ZG0701X,"Specialist/Technologist, Other Graphics Methods","Technologists, Technicians & Other Technical Service Providers" -246ZG1000X,"Specialist/Technologist, Other Geneticist, Medical (PhD)","Technologists, Technicians & Other Technical Service Providers" -246ZI1000X,"Specialist/Technologist, Other Illustration, Medical","Technologists, Technicians & Other Technical Service Providers" -246ZN0300X,"Specialist/Technologist, Other Nephrology","Technologists, Technicians & Other Technical Service Providers" -246ZS0410X,"Specialist/Technologist, Other Surgical Technologist","Technologists, Technicians & Other Technical Service Providers" -246ZX2200X,"Specialist/Technologist, Other Orthopedic Assistant","Technologists, Technicians & Other Technical Service Providers" -247000000X,"Technician, Health Information ","Technologists, Technicians & Other Technical Service Providers" -2470A2800X,"Technician, Health Information Assistant Record Technician","Technologists, Technicians & Other Technical Service Providers" -247100000X,Radiologic Technologist ,"Technologists, Technicians & Other Technical Service Providers" -2471B0102X,Radiologic Technologist Bone Densitometry,"Technologists, Technicians & Other Technical Service Providers" -2471C1101X,Radiologic Technologist Cardiovascular-Interventional Technology,"Technologists, Technicians & Other Technical Service Providers" -2471C1106X,Radiologic Technologist Cardiac-Interventional Technology,"Technologists, Technicians & Other Technical Service Providers" -2471C3401X,Radiologic Technologist Computed Tomography,"Technologists, Technicians & Other Technical Service Providers" -2471C3402X,Radiologic Technologist Radiography,"Technologists, Technicians & Other Technical Service Providers" -2471M1202X,Radiologic Technologist Magnetic Resonance Imaging,"Technologists, Technicians & Other Technical Service Providers" -2471M2300X,Radiologic Technologist Mammography,"Technologists, Technicians & Other Technical Service Providers" -2471N0900X,Radiologic Technologist Nuclear Medicine Technology,"Technologists, Technicians & Other Technical Service Providers" -2471Q0001X,Radiologic Technologist Quality Management,"Technologists, Technicians & Other Technical Service Providers" -2471R0002X,Radiologic Technologist Radiation Therapy,"Technologists, Technicians & Other Technical Service Providers" -2471S1302X,Radiologic Technologist Sonography,"Technologists, Technicians & Other Technical Service Providers" -2471V0105X,Radiologic Technologist Vascular Sonography,"Technologists, Technicians & Other Technical Service Providers" -2471V0106X,Radiologic Technologist Vascular-Interventional Technology,"Technologists, Technicians & Other Technical Service Providers" -247200000X,"Technician, Other ","Technologists, Technicians & Other Technical Service Providers" -2472B0301X,"Technician, Other Biomedical Engineering","Technologists, Technicians & Other Technical Service Providers" -2472D0500X,"Technician, Other Darkroom","Technologists, Technicians & Other Technical Service Providers" -2472E0500X,"Technician, Other EEG","Technologists, Technicians & Other Technical Service Providers" -2472R0900X,"Technician, Other Renal Dialysis","Technologists, Technicians & Other Technical Service Providers" -2472V0600X,"Technician, Other Veterinary","Technologists, Technicians & Other Technical Service Providers" -247ZC0005X,"Pathology Clinical Laboratory Director, Non-physician","Technologists, Technicians & Other Technical Service Providers" +221700000X,Art Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +222Q00000X,Developmental Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +222Z00000X,Orthotist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +224900000X,Mastectomy Fitter ,Respiratory Developmental Rehabilitative and Restorative Service Providers +224L00000X,Pedorthist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +224P00000X,Prosthetist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +224Y00000X,Clinical Exercise Physiologist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +224Z00000X,Occupational Therapy Assistant ,Respiratory Developmental Rehabilitative and Restorative Service Providers +224ZE0001X,Occupational Therapy Assistant Environmental Modification,Respiratory Developmental Rehabilitative and Restorative Service Providers +224ZF0002X,Occupational Therapy Assistant Feeding Eating & Swallowing,Respiratory Developmental Rehabilitative and Restorative Service Providers +224ZL0004X,Occupational Therapy Assistant Low Vision,Respiratory Developmental Rehabilitative and Restorative Service Providers +224ZR0403X,Occupational Therapy Assistant Driving and Community Mobility,Respiratory Developmental Rehabilitative and Restorative Service Providers +225000000X,Orthotic Fitter ,Respiratory Developmental Rehabilitative and Restorative Service Providers +225100000X,Physical Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +2251C2600X,Physical Therapist Cardiopulmonary,Respiratory Developmental Rehabilitative and Restorative Service Providers +2251E1200X,Physical Therapist Ergonomics,Respiratory Developmental Rehabilitative and Restorative Service Providers +2251E1300X,Physical Therapist Electrophysiology Clinical,Respiratory Developmental Rehabilitative and Restorative Service Providers +2251G0304X,Physical Therapist Geriatrics,Respiratory Developmental Rehabilitative and Restorative Service Providers +2251H1200X,Physical Therapist Hand,Respiratory Developmental Rehabilitative and Restorative Service Providers +2251H1300X,Physical Therapist Human Factors,Respiratory Developmental Rehabilitative and Restorative Service Providers +2251N0400X,Physical Therapist Neurology,Respiratory Developmental Rehabilitative and Restorative Service Providers +2251P0200X,Physical Therapist Pediatrics,Respiratory Developmental Rehabilitative and Restorative Service Providers +2251S0007X,Physical Therapist Sports,Respiratory Developmental Rehabilitative and Restorative Service Providers +2251X0800X,Physical Therapist Orthopedic,Respiratory Developmental Rehabilitative and Restorative Service Providers +225200000X,Physical Therapy Assistant ,Respiratory Developmental Rehabilitative and Restorative Service Providers +225400000X,Rehabilitation Practitioner ,Respiratory Developmental Rehabilitative and Restorative Service Providers +225500000X,Specialist/Technologist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +2255A2300X,Specialist/Technologist Athletic Trainer,Respiratory Developmental Rehabilitative and Restorative Service Providers +2255R0406X,Specialist/Technologist Rehabilitation Blind,Respiratory Developmental Rehabilitative and Restorative Service Providers +225600000X,Dance Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +225700000X,Massage Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +225800000X,Recreation Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +225A00000X,Music Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +225B00000X,Pulmonary Function Technologist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +225C00000X,Rehabilitation Counselor ,Respiratory Developmental Rehabilitative and Restorative Service Providers +225CA2400X,Rehabilitation Counselor Assistive Technology Practitioner,Respiratory Developmental Rehabilitative and Restorative Service Providers +225CA2500X,Rehabilitation Counselor Assistive Technology Supplier,Respiratory Developmental Rehabilitative and Restorative Service Providers +225CX0006X,Rehabilitation Counselor Orientation and Mobility Training Provider,Respiratory Developmental Rehabilitative and Restorative Service Providers +225X00000X,Occupational Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +225XE0001X,Occupational Therapist Environmental Modification,Respiratory Developmental Rehabilitative and Restorative Service Providers +225XE1200X,Occupational Therapist Ergonomics,Respiratory Developmental Rehabilitative and Restorative Service Providers +225XF0002X,Occupational Therapist Feeding Eating & Swallowing,Respiratory Developmental Rehabilitative and Restorative Service Providers +225XG0600X,Occupational Therapist Gerontology,Respiratory Developmental Rehabilitative and Restorative Service Providers +225XH1200X,Occupational Therapist Hand,Respiratory Developmental Rehabilitative and Restorative Service Providers +225XH1300X,Occupational Therapist Human Factors,Respiratory Developmental Rehabilitative and Restorative Service Providers +225XL0004X,Occupational Therapist Low Vision,Respiratory Developmental Rehabilitative and Restorative Service Providers +225XM0800X,Occupational Therapist Mental Health,Respiratory Developmental Rehabilitative and Restorative Service Providers +225XN1300X,Occupational Therapist Neurorehabilitation,Respiratory Developmental Rehabilitative and Restorative Service Providers +225XP0019X,Occupational Therapist Physical Rehabilitation,Respiratory Developmental Rehabilitative and Restorative Service Providers +225XP0200X,Occupational Therapist Pediatrics,Respiratory Developmental Rehabilitative and Restorative Service Providers +225XR0403X,Occupational Therapist Driving and Community Mobility,Respiratory Developmental Rehabilitative and Restorative Service Providers +226000000X,Recreational Therapist Assistant ,Respiratory Developmental Rehabilitative and Restorative Service Providers +226300000X,Kinesiotherapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +227800000X,Respiratory Therapist Certified ,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278C0205X,Respiratory Therapist Certified Critical Care,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278E0002X,Respiratory Therapist Certified Emergency Care,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278E1000X,Respiratory Therapist Certified Educational,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278G0305X,Respiratory Therapist Certified Geriatric Care,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278G1100X,Respiratory Therapist Certified General Care,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278H0200X,Respiratory Therapist Certified Home Health,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278P1004X,Respiratory Therapist Certified Pulmonary Diagnostics,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278P1005X,Respiratory Therapist Certified Pulmonary Rehabilitation,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278P1006X,Respiratory Therapist Certified Pulmonary Function Technologist,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278P3800X,Respiratory Therapist Certified Palliative/Hospice,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278P3900X,Respiratory Therapist Certified Neonatal/Pediatrics,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278P4000X,Respiratory Therapist Certified Patient Transport,Respiratory Developmental Rehabilitative and Restorative Service Providers +2278S1500X,Respiratory Therapist Certified SNF/Subacute Care,Respiratory Developmental Rehabilitative and Restorative Service Providers +227900000X,Respiratory Therapist Registered ,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279C0205X,Respiratory Therapist Registered Critical Care,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279E0002X,Respiratory Therapist Registered Emergency Care,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279E1000X,Respiratory Therapist Registered Educational,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279G0305X,Respiratory Therapist Registered Geriatric Care,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279G1100X,Respiratory Therapist Registered General Care,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279H0200X,Respiratory Therapist Registered Home Health,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279P1004X,Respiratory Therapist Registered Pulmonary Diagnostics,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279P1005X,Respiratory Therapist Registered Pulmonary Rehabilitation,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279P1006X,Respiratory Therapist Registered Pulmonary Function Technologist,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279P3800X,Respiratory Therapist Registered Palliative/Hospice,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279P3900X,Respiratory Therapist Registered Neonatal/Pediatrics,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279P4000X,Respiratory Therapist Registered Patient Transport,Respiratory Developmental Rehabilitative and Restorative Service Providers +2279S1500X,Respiratory Therapist Registered SNF/Subacute Care,Respiratory Developmental Rehabilitative and Restorative Service Providers +229N00000X,Anaplastologist ,Respiratory Developmental Rehabilitative and Restorative Service Providers +231H00000X,Audiologist ,Speech Language and Hearing Service Providers +231HA2400X,Audiologist Assistive Technology Practitioner,Speech Language and Hearing Service Providers +231HA2500X,Audiologist Assistive Technology Supplier,Speech Language and Hearing Service Providers +235500000X,Specialist/Technologist ,Speech Language and Hearing Service Providers +2355A2700X,Specialist/Technologist Audiology Assistant,Speech Language and Hearing Service Providers +2355S0801X,Specialist/Technologist Speech-Language Assistant,Speech Language and Hearing Service Providers +235Z00000X,Speech-Language Pathologist ,Speech Language and Hearing Service Providers +237600000X,Audiologist-Hearing Aid Fitter ,Speech Language and Hearing Service Providers +237700000X,Hearing Instrument Specialist ,Speech Language and Hearing Service Providers +242T00000X,Perfusionist ,Technologists Technicians & Other Technical Service Providers +243U00000X,Radiology Practitioner Assistant ,Technologists Technicians & Other Technical Service Providers +246Q00000X,Spec/Tech Pathology ,Technologists Technicians & Other Technical Service Providers +246QB0000X,Spec/Tech Pathology Blood Banking,Technologists Technicians & Other Technical Service Providers +246QC1000X,Spec/Tech Pathology Chemistry,Technologists Technicians & Other Technical Service Providers +246QC2700X,Spec/Tech Pathology Cytotechnology,Technologists Technicians & Other Technical Service Providers +246QH0000X,Spec/Tech Pathology Hematology,Technologists Technicians & Other Technical Service Providers +246QH0401X,Spec/Tech Pathology Hemapheresis Practitioner,Technologists Technicians & Other Technical Service Providers +246QH0600X,Spec/Tech Pathology Histology,Technologists Technicians & Other Technical Service Providers +246QI0000X,Spec/Tech Pathology Immunology,Technologists Technicians & Other Technical Service Providers +246QL0900X,Spec/Tech Pathology Laboratory Management,Technologists Technicians & Other Technical Service Providers +246QL0901X,Spec/Tech Pathology Laboratory Management Diplomate,Technologists Technicians & Other Technical Service Providers +246QM0706X,Spec/Tech Pathology Medical Technologist,Technologists Technicians & Other Technical Service Providers +246QM0900X,Spec/Tech Pathology Microbiology,Technologists Technicians & Other Technical Service Providers +246R00000X,Technician Pathology ,Technologists Technicians & Other Technical Service Providers +246RH0600X,Technician Pathology Histology,Technologists Technicians & Other Technical Service Providers +246RM2200X,Technician Pathology Medical Laboratory,Technologists Technicians & Other Technical Service Providers +246RP1900X,Technician Pathology Phlebotomy,Technologists Technicians & Other Technical Service Providers +246W00000X,Technician Cardiology ,Technologists Technicians & Other Technical Service Providers +246X00000X,Spec/Tech Cardiovascular ,Technologists Technicians & Other Technical Service Providers +246XC2901X,Spec/Tech Cardiovascular Cardiovascular Invasive Specialist,Technologists Technicians & Other Technical Service Providers +246XC2903X,Spec/Tech Cardiovascular Vascular Specialist,Technologists Technicians & Other Technical Service Providers +246XS1301X,Spec/Tech Cardiovascular Sonography,Technologists Technicians & Other Technical Service Providers +246Y00000X,Spec/Tech Health Info ,Technologists Technicians & Other Technical Service Providers +246YC3301X,Spec/Tech Health Info Coding Specialist Hospital Based,Technologists Technicians & Other Technical Service Providers +246YC3302X,Spec/Tech Health Info Coding Specialist Physician Office Based,Technologists Technicians & Other Technical Service Providers +246YR1600X,Spec/Tech Health Info Registered Record Administrator,Technologists Technicians & Other Technical Service Providers +246Z00000X,Specialist/Technologist Other ,Technologists Technicians & Other Technical Service Providers +246ZA2600X,Specialist/Technologist Other Art Medical,Technologists Technicians & Other Technical Service Providers +246ZB0301X,Specialist/Technologist Other Biomedical Engineering,Technologists Technicians & Other Technical Service Providers +246ZB0302X,Specialist/Technologist Other Biomedical Photographer,Technologists Technicians & Other Technical Service Providers +246ZB0500X,Specialist/Technologist Other Biochemist,Technologists Technicians & Other Technical Service Providers +246ZB0600X,Specialist/Technologist Other Biostatistician,Technologists Technicians & Other Technical Service Providers +246ZC0007X,Specialist/Technologist Other Surgical Assistant,Technologists Technicians & Other Technical Service Providers +246ZE0500X,Specialist/Technologist Other EEG,Technologists Technicians & Other Technical Service Providers +246ZE0600X,Specialist/Technologist Other Electroneurodiagnostic,Technologists Technicians & Other Technical Service Providers +246ZG0701X,Specialist/Technologist Other Graphics Methods,Technologists Technicians & Other Technical Service Providers +246ZG1000X,Specialist/Technologist Other Geneticist Medical (PhD),Technologists Technicians & Other Technical Service Providers +246ZI1000X,Specialist/Technologist Other Illustration Medical,Technologists Technicians & Other Technical Service Providers +246ZN0300X,Specialist/Technologist Other Nephrology,Technologists Technicians & Other Technical Service Providers +246ZS0410X,Specialist/Technologist Other Surgical Technologist,Technologists Technicians & Other Technical Service Providers +246ZX2200X,Specialist/Technologist Other Orthopedic Assistant,Technologists Technicians & Other Technical Service Providers +247000000X,Technician Health Information ,Technologists Technicians & Other Technical Service Providers +2470A2800X,Technician Health Information Assistant Record Technician,Technologists Technicians & Other Technical Service Providers +247100000X,Radiologic Technologist ,Technologists Technicians & Other Technical Service Providers +2471B0102X,Radiologic Technologist Bone Densitometry,Technologists Technicians & Other Technical Service Providers +2471C1101X,Radiologic Technologist Cardiovascular-Interventional Technology,Technologists Technicians & Other Technical Service Providers +2471C1106X,Radiologic Technologist Cardiac-Interventional Technology,Technologists Technicians & Other Technical Service Providers +2471C3401X,Radiologic Technologist Computed Tomography,Technologists Technicians & Other Technical Service Providers +2471C3402X,Radiologic Technologist Radiography,Technologists Technicians & Other Technical Service Providers +2471M1202X,Radiologic Technologist Magnetic Resonance Imaging,Technologists Technicians & Other Technical Service Providers +2471M2300X,Radiologic Technologist Mammography,Technologists Technicians & Other Technical Service Providers +2471N0900X,Radiologic Technologist Nuclear Medicine Technology,Technologists Technicians & Other Technical Service Providers +2471Q0001X,Radiologic Technologist Quality Management,Technologists Technicians & Other Technical Service Providers +2471R0002X,Radiologic Technologist Radiation Therapy,Technologists Technicians & Other Technical Service Providers +2471S1302X,Radiologic Technologist Sonography,Technologists Technicians & Other Technical Service Providers +2471V0105X,Radiologic Technologist Vascular Sonography,Technologists Technicians & Other Technical Service Providers +2471V0106X,Radiologic Technologist Vascular-Interventional Technology,Technologists Technicians & Other Technical Service Providers +247200000X,Technician Other ,Technologists Technicians & Other Technical Service Providers +2472B0301X,Technician Other Biomedical Engineering,Technologists Technicians & Other Technical Service Providers +2472D0500X,Technician Other Darkroom,Technologists Technicians & Other Technical Service Providers +2472E0500X,Technician Other EEG,Technologists Technicians & Other Technical Service Providers +2472R0900X,Technician Other Renal Dialysis,Technologists Technicians & Other Technical Service Providers +2472V0600X,Technician Other Veterinary,Technologists Technicians & Other Technical Service Providers +247ZC0005X,Pathology Clinical Laboratory Director Non-physician,Technologists Technicians & Other Technical Service Providers 251300000X,Local Education Agency (LEA) ,Agencies 251B00000X,Case Management ,Agencies -251C00000X,"Day Training, Developmentally Disabled Services ",Agencies +251C00000X,Day Training Developmentally Disabled Services ,Agencies 251E00000X,Home Health ,Agencies 251F00000X,Home Infusion ,Agencies -251G00000X,"Hospice Care, Community Based ",Agencies +251G00000X,Hospice Care Community Based ,Agencies 251J00000X,Nursing Care ,Agencies 251K00000X,Public Health or Welfare ,Agencies 251S00000X,Community/Behavioral Health ,Agencies @@ -628,7 +628,7 @@ Code,Descriptive_Text,Grouping 261QE0002X,Clinic/Center Emergency Care,Ambulatory Health Care Facilities 261QE0700X,Clinic/Center End-Stage Renal Disease (ESRD) Treatment,Ambulatory Health Care Facilities 261QE0800X,Clinic/Center Endoscopy,Ambulatory Health Care Facilities -261QF0050X,"Clinic/Center Family Planning, Non-Surgical",Ambulatory Health Care Facilities +261QF0050X,Clinic/Center Family Planning Non-Surgical,Ambulatory Health Care Facilities 261QF0400X,Clinic/Center Federally Qualified Health Center (FQHC),Ambulatory Health Care Facilities 261QG0250X,Clinic/Center Genetics,Ambulatory Health Care Facilities 261QH0100X,Clinic/Center Health Service,Ambulatory Health Care Facilities @@ -648,21 +648,21 @@ Code,Descriptive_Text,Grouping 261QM2500X,Clinic/Center Medical Specialty,Ambulatory Health Care Facilities 261QM2800X,Clinic/Center Methadone Clinic,Ambulatory Health Care Facilities 261QM3000X,Clinic/Center Medically Fragile Intants and Children Day Care,Ambulatory Health Care Facilities -261QP0904X,"Clinic/Center Public Health, Federal",Ambulatory Health Care Facilities -261QP0905X,"Clinic/Center Public Health, State or Local",Ambulatory Health Care Facilities +261QP0904X,Clinic/Center Public Health Federal,Ambulatory Health Care Facilities +261QP0905X,Clinic/Center Public Health State or Local,Ambulatory Health Care Facilities 261QP1100X,Clinic/Center Podiatric,Ambulatory Health Care Facilities 261QP2000X,Clinic/Center Physical Therapy,Ambulatory Health Care Facilities 261QP2300X,Clinic/Center Primary Care,Ambulatory Health Care Facilities 261QP2400X,Clinic/Center Prison Health,Ambulatory Health Care Facilities 261QP3300X,Clinic/Center Pain,Ambulatory Health Care Facilities 261QR0200X,Clinic/Center Radiology,Ambulatory Health Care Facilities -261QR0206X,"Clinic/Center Radiology, Mammography",Ambulatory Health Care Facilities -261QR0207X,"Clinic/Center Radiology, Mobile Mammography",Ambulatory Health Care Facilities -261QR0208X,"Clinic/Center Radiology, Mobile",Ambulatory Health Care Facilities +261QR0206X,Clinic/Center Radiology Mammography,Ambulatory Health Care Facilities +261QR0207X,Clinic/Center Radiology Mobile Mammography,Ambulatory Health Care Facilities +261QR0208X,Clinic/Center Radiology Mobile,Ambulatory Health Care Facilities 261QR0400X,Clinic/Center Rehabilitation,Ambulatory Health Care Facilities -261QR0401X,"Clinic/Center Rehabilitation, Comprehensive Outpatient Rehabilitation Facility (CORF)",Ambulatory Health Care Facilities -261QR0404X,"Clinic/Center Rehabilitation, Cardiac Facilities",Ambulatory Health Care Facilities -261QR0405X,"Clinic/Center Rehabilitation, Substance Use Disorder",Ambulatory Health Care Facilities +261QR0401X,Clinic/Center Rehabilitation Comprehensive Outpatient Rehabilitation Facility (CORF),Ambulatory Health Care Facilities +261QR0404X,Clinic/Center Rehabilitation Cardiac Facilities,Ambulatory Health Care Facilities +261QR0405X,Clinic/Center Rehabilitation Substance Use Disorder,Ambulatory Health Care Facilities 261QR0800X,Clinic/Center Recovery Care,Ambulatory Health Care Facilities 261QR1100X,Clinic/Center Research,Ambulatory Health Care Facilities 261QR1300X,Clinic/Center Rural Health,Ambulatory Health Care Facilities @@ -674,12 +674,12 @@ Code,Descriptive_Text,Grouping 261QV0200X,Clinic/Center VA,Ambulatory Health Care Facilities 261QX0100X,Clinic/Center Occupational Medicine,Ambulatory Health Care Facilities 261QX0200X,Clinic/Center Oncology,Ambulatory Health Care Facilities -261QX0203X,"Clinic/Center Oncology, Radiation",Ambulatory Health Care Facilities +261QX0203X,Clinic/Center Oncology Radiation,Ambulatory Health Care Facilities 273100000X,Epilepsy Unit ,Hospital Units 273R00000X,Psychiatric Unit ,Hospital Units 273Y00000X,Rehabilitation Unit ,Hospital Units 275N00000X,Medicare Defined Swing Bed Unit ,Hospital Units -276400000X,"Rehabilitation, Substance Use Disorder Unit ",Hospital Units +276400000X,Rehabilitation Substance Use Disorder Unit ,Hospital Units 281P00000X,Chronic Disease Hospital ,Hospitals 281PC2000X,Chronic Disease Hospital Children,Hospitals 282E00000X,Long Term Care Hospital ,Hospitals @@ -707,26 +707,26 @@ Code,Descriptive_Text,Grouping 305R00000X,Preferred Provider Organization ,Managed Care Organizations 305S00000X,Point of Service ,Managed Care Organizations 310400000X,Assisted Living Facility ,Nursing & Custodial Care Facilities -3104A0625X,"Assisted Living Facility Assisted Living, Mental Illness",Nursing & Custodial Care Facilities -3104A0630X,"Assisted Living Facility Assisted Living, Behavioral Disturbances",Nursing & Custodial Care Facilities -310500000X,"Intermediate Care Facility, Mental Illness ",Nursing & Custodial Care Facilities +3104A0625X,Assisted Living Facility Assisted Living Mental Illness,Nursing & Custodial Care Facilities +3104A0630X,Assisted Living Facility Assisted Living Behavioral Disturbances,Nursing & Custodial Care Facilities +310500000X,Intermediate Care Facility Mental Illness ,Nursing & Custodial Care Facilities 311500000X,Alzheimer Center (Dementia Center) ,Nursing & Custodial Care Facilities 311Z00000X,Custodial Care Facility ,Nursing & Custodial Care Facilities 311ZA0620X,Custodial Care Facility Adult Care Home,Nursing & Custodial Care Facilities 313M00000X,Nursing Facility/Intermediate Care Facility ,Nursing & Custodial Care Facilities 314000000X,Skilled Nursing Facility ,Nursing & Custodial Care Facilities -3140N1450X,"Skilled Nursing Facility Nursing Care, Pediatric",Nursing & Custodial Care Facilities -315D00000X,"Hospice, Inpatient ",Nursing & Custodial Care Facilities -315P00000X,"Intermediate Care Facility, Mentally Retarded ",Nursing & Custodial Care Facilities +3140N1450X,Skilled Nursing Facility Nursing Care Pediatric,Nursing & Custodial Care Facilities +315D00000X,Hospice Inpatient ,Nursing & Custodial Care Facilities +315P00000X,Intermediate Care Facility Mentally Retarded ,Nursing & Custodial Care Facilities 317400000X,Christian Science Facility ,Nursing & Custodial Care Facilities -320600000X,"Residential Treatment Facility, Mental Retardation and/or Developmental Disabilities ",Residential Treatment Facilities -320700000X,"Residential Treatment Facility, Physical Disabilities ",Residential Treatment Facilities -320800000X,"Community Based Residential Treatment Facility, Mental Illness ",Residential Treatment Facilities -320900000X,"Community Based Residential Treatment, Mental Retardation and/or Developmental Disabilities ",Residential Treatment Facilities -322D00000X,"Residential Treatment Facility, Emotionally Disturbed Children ",Residential Treatment Facilities +320600000X,Residential Treatment Facility Mental Retardation and/or Developmental Disabilities ,Residential Treatment Facilities +320700000X,Residential Treatment Facility Physical Disabilities ,Residential Treatment Facilities +320800000X,Community Based Residential Treatment Facility Mental Illness ,Residential Treatment Facilities +320900000X,Community Based Residential Treatment Mental Retardation and/or Developmental Disabilities ,Residential Treatment Facilities +322D00000X,Residential Treatment Facility Emotionally Disturbed Children ,Residential Treatment Facilities 323P00000X,Psychiatric Residential Treatment Facility ,Residential Treatment Facilities 324500000X,Substance Abuse Rehabilitation Facility ,Residential Treatment Facilities -3245S0500X,"Substance Abuse Rehabilitation Facility Substance Abuse Treatment, Children",Residential Treatment Facilities +3245S0500X,Substance Abuse Rehabilitation Facility Substance Abuse Treatment Children,Residential Treatment Facilities 331L00000X,Blood Bank ,Suppliers 332000000X,Military/U.S. Coast Guard Pharmacy ,Suppliers 332100000X,Department of Veterans Affairs (VA) Pharmacy ,Suppliers @@ -739,7 +739,7 @@ Code,Descriptive_Text,Grouping 332BP3500X,Durable Medical Equipment & Medical Supplies Parenteral & Enteral Nutrition,Suppliers 332BX2000X,Durable Medical Equipment & Medical Supplies Oxygen Equipment & Supplies,Suppliers 332G00000X,Eye Bank ,Suppliers -332H00000X,"Eyewear Supplier (Equipment, not the service) ",Suppliers +332H00000X,Eyewear Supplier (Equipment not the service) ,Suppliers 332S00000X,Hearing Aid Equipment ,Suppliers 332U00000X,Home Delivered Meals ,Suppliers 333300000X,Emergency Response System Companies ,Suppliers @@ -763,9 +763,9 @@ Code,Descriptive_Text,Grouping 3416L0300X,Ambulance Land Transport,Transportation Services 3416S0300X,Ambulance Water Transport,Transportation Services 341800000X,Military/U.S. Coast Guard Transport ,Transportation Services -3418M1110X,"Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance, Ground Transport",Transportation Services -3418M1120X,"Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance, Air Transport",Transportation Services -3418M1130X,"Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance, Water Transport",Transportation Services +3418M1110X,Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance Ground Transport,Transportation Services +3418M1120X,Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance Air Transport,Transportation Services +3418M1130X,Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance Water Transport,Transportation Services 343800000X,Secured Medical Transport (VAN) ,Transportation Services 343900000X,Non-emergency Medical Transport (VAN) ,Transportation Services 344600000X,Taxi ,Transportation Services @@ -785,9 +785,9 @@ Code,Descriptive_Text,Grouping 363LF0000X,Nurse Practitioner Family,Physician Assistants & Advanced Practice Nursing Providers 363LG0600X,Nurse Practitioner Gerontology,Physician Assistants & Advanced Practice Nursing Providers 363LN0000X,Nurse Practitioner Neonatal,Physician Assistants & Advanced Practice Nursing Providers -363LN0005X,"Nurse Practitioner Neonatal, Critical Care",Physician Assistants & Advanced Practice Nursing Providers +363LN0005X,Nurse Practitioner Neonatal Critical Care,Physician Assistants & Advanced Practice Nursing Providers 363LP0200X,Nurse Practitioner Pediatrics,Physician Assistants & Advanced Practice Nursing Providers -363LP0222X,"Nurse Practitioner Pediatrics, Critical Care",Physician Assistants & Advanced Practice Nursing Providers +363LP0222X,Nurse Practitioner Pediatrics Critical Care,Physician Assistants & Advanced Practice Nursing Providers 363LP0808X,Nurse Practitioner Psych/Mental Health,Physician Assistants & Advanced Practice Nursing Providers 363LP1700X,Nurse Practitioner Perinatal,Physician Assistants & Advanced Practice Nursing Providers 363LP2300X,Nurse Practitioner Primary Care,Physician Assistants & Advanced Practice Nursing Providers @@ -813,13 +813,13 @@ Code,Descriptive_Text,Grouping 364SN0000X,Clinical Nurse Specialist Neonatal,Physician Assistants & Advanced Practice Nursing Providers 364SN0800X,Clinical Nurse Specialist Neuroscience,Physician Assistants & Advanced Practice Nursing Providers 364SP0200X,Clinical Nurse Specialist Pediatrics,Physician Assistants & Advanced Practice Nursing Providers -364SP0807X,"Clinical Nurse Specialist Psych/Mental Health, Child & Adolescent",Physician Assistants & Advanced Practice Nursing Providers +364SP0807X,Clinical Nurse Specialist Psych/Mental Health Child & Adolescent,Physician Assistants & Advanced Practice Nursing Providers 364SP0808X,Clinical Nurse Specialist Psych/Mental Health,Physician Assistants & Advanced Practice Nursing Providers -364SP0809X,"Clinical Nurse Specialist Psych/Mental Health, Adult",Physician Assistants & Advanced Practice Nursing Providers -364SP0810X,"Clinical Nurse Specialist Psych/Mental Health, Child & Family",Physician Assistants & Advanced Practice Nursing Providers -364SP0811X,"Clinical Nurse Specialist Psych/Mental Health, Chronically Ill",Physician Assistants & Advanced Practice Nursing Providers -364SP0812X,"Clinical Nurse Specialist Psych/Mental Health, Community",Physician Assistants & Advanced Practice Nursing Providers -364SP0813X,"Clinical Nurse Specialist Psych/Mental Health, Geropsychiatric",Physician Assistants & Advanced Practice Nursing Providers +364SP0809X,Clinical Nurse Specialist Psych/Mental Health Adult,Physician Assistants & Advanced Practice Nursing Providers +364SP0810X,Clinical Nurse Specialist Psych/Mental Health Child & Family,Physician Assistants & Advanced Practice Nursing Providers +364SP0811X,Clinical Nurse Specialist Psych/Mental Health Chronically Ill,Physician Assistants & Advanced Practice Nursing Providers +364SP0812X,Clinical Nurse Specialist Psych/Mental Health Community,Physician Assistants & Advanced Practice Nursing Providers +364SP0813X,Clinical Nurse Specialist Psych/Mental Health Geropsychiatric,Physician Assistants & Advanced Practice Nursing Providers 364SP1700X,Clinical Nurse Specialist Perinatal,Physician Assistants & Advanced Practice Nursing Providers 364SP2800X,Clinical Nurse Specialist Perioperative,Physician Assistants & Advanced Practice Nursing Providers 364SR0400X,Clinical Nurse Specialist Rehabilitation,Physician Assistants & Advanced Practice Nursing Providers @@ -828,8 +828,8 @@ Code,Descriptive_Text,Grouping 364SW0102X,Clinical Nurse Specialist Women's Health,Physician Assistants & Advanced Practice Nursing Providers 364SX0106X,Clinical Nurse Specialist Occupational Health,Physician Assistants & Advanced Practice Nursing Providers 364SX0200X,Clinical Nurse Specialist Oncology,Physician Assistants & Advanced Practice Nursing Providers -364SX0204X,"Clinical Nurse Specialist Oncology, Pediatrics",Physician Assistants & Advanced Practice Nursing Providers -367500000X,"Nurse Anesthetist, Certified Registered ",Physician Assistants & Advanced Practice Nursing Providers +364SX0204X,Clinical Nurse Specialist Oncology Pediatrics,Physician Assistants & Advanced Practice Nursing Providers +367500000X,Nurse Anesthetist Certified Registered ,Physician Assistants & Advanced Practice Nursing Providers 367A00000X,Advanced Practice Midwife ,Physician Assistants & Advanced Practice Nursing Providers 367H00000X,Anesthesiologist Assistant ,Physician Assistants & Advanced Practice Nursing Providers 372500000X,Chore Provider ,Nursing Service Related Providers @@ -847,10 +847,10 @@ Code,Descriptive_Text,Grouping 376K00000X,Nurse's Aide ,Nursing Service Related Providers 385H00000X,Respite Care ,Respite Care Facility 385HR2050X,Respite Care Respite Care Camp,Respite Care Facility -385HR2055X,"Respite Care Respite Care, Mental Illness, Child",Respite Care Facility -385HR2060X,"Respite Care Respite Care, Mental Retardation and/or Developmental Disabilities, Child",Respite Care Facility -385HR2065X,"Respite Care Respite Care, Physical Disabilities, Child",Respite Care Facility -390200000X,Student in an Organized Health Care Education/Training Program ,"Student, Health Care" +385HR2055X,Respite Care Respite Care Mental Illness Child,Respite Care Facility +385HR2060X,Respite Care Respite Care Mental Retardation and/or Developmental Disabilities Child,Respite Care Facility +385HR2065X,Respite Care Respite Care Physical Disabilities Child,Respite Care Facility +390200000X,Student in an Organized Health Care Education/Training Program ,Student Health Care 405300000X,Prevention Professional ,Other Service Providers NI,No information, UN,Unknown, diff --git a/i2p_tasks.py b/i2p_tasks.py index 6cf221c..b1a2e2f 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -307,7 +307,7 @@ def fetch(self): with open(self.dl_path + self.npi_zip, 'wb') as fout: fout.write(r.read()) - subprocess.call('unzip', '-o', self.dl_path + self.npi_zip) + subprocess.call(['unzip', '-o', self.dl_path + self.npi_zip]) def extract(self): self.expectedRecords = 0 From 2b207db4424418b62c630d86fc1b9e67552c77e5 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 10 May 2018 17:30:29 -0500 Subject: [PATCH 392/507] Fixing typo in csv name --- i2p_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index b1a2e2f..66201b5 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -287,7 +287,7 @@ class downloadNPI(CDMStatusTask): dl_path = '/d1/npi/' load_path = 'curated_data/' npi_zip = 'NPPES_Data_Dissemination_April_2018.zip' - npi_csv = 'npidata_pfile_20050523 - 20180408.csv' + npi_csv = 'npidata_pfile_20050523-20180408.csv' specialty_csv = 'provider_specialty_map.csv' taxonomy_col = 'Healthcare Provider Taxonomy Code_' From 13c83045f9960af18a2657cea71230487adf9de3 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 11 May 2018 09:40:13 -0500 Subject: [PATCH 393/507] Removing unicode characters --- curated_data/provider_specialty_code.csv | 448 +++++++++++------------ 1 file changed, 224 insertions(+), 224 deletions(-) diff --git a/curated_data/provider_specialty_code.csv b/curated_data/provider_specialty_code.csv index b9f9ead..b502b3f 100644 --- a/curated_data/provider_specialty_code.csv +++ b/curated_data/provider_specialty_code.csv @@ -71,16 +71,16 @@ Code,Descriptive_Text,Grouping 126900000X,Dental Laboratory Technician ,Dental Providers 132700000X,Dietary Manager ,Dietary & Nutritional Service Providers 133N00000X,Nutritionist ,Dietary & Nutritional Service Providers -133NN1002X,Nutritionist Nutrition Education,Dietary & Nutritional Service Providers -133V00000X,Dietitian Registered ,Dietary & Nutritional Service Providers -133VN1004X,Dietitian Registered Nutrition Pediatric,Dietary & Nutritional Service Providers -133VN1005X,Dietitian Registered Nutrition Renal,Dietary & Nutritional Service Providers -133VN1006X,Dietitian Registered Nutrition Metabolic,Dietary & Nutritional Service Providers -136A00000X,Dietetic Technician Registered ,Dietary & Nutritional Service Providers +133NN1002X,"Nutritionist Nutrition, Education",Dietary & Nutritional Service Providers +133V00000X,"Dietitian, Registered ",Dietary & Nutritional Service Providers +133VN1004X,"Dietitian, Registered Nutrition, Pediatric",Dietary & Nutritional Service Providers +133VN1005X,"Dietitian, Registered Nutrition, Renal",Dietary & Nutritional Service Providers +133VN1006X,"Dietitian, Registered Nutrition, Metabolic",Dietary & Nutritional Service Providers +136A00000X,"Dietetic Technician, Registered ",Dietary & Nutritional Service Providers 146D00000X,Personal Emergency Response Attendant ,Emergency Medical Service Providers -146L00000X,Emergency Medical Technician Paramedic ,Emergency Medical Service Providers -146M00000X,Emergency Medical Technician Intermediate ,Emergency Medical Service Providers -146N00000X,Emergency Medical Technician Basic ,Emergency Medical Service Providers +146L00000X,"Emergency Medical Technician, Paramedic ",Emergency Medical Service Providers +146M00000X,"Emergency Medical Technician, Intermediate ",Emergency Medical Service Providers +146N00000X,"Emergency Medical Technician, Basic ",Emergency Medical Service Providers 152W00000X,Optometrist ,Eye and Vision Services Providers 152WC0802X,Optometrist Corneal and Contact Management,Eye and Vision Services Providers 152WL0500X,Optometrist Low Vision Rehabilitation,Eye and Vision Services Providers @@ -109,7 +109,7 @@ Code,Descriptive_Text,Grouping 163WC2100X,Registered Nurse Continence Care,Nursing Service Providers 163WC3500X,Registered Nurse Cardiac Rehabilitation,Nursing Service Providers 163WD0400X,Registered Nurse Diabetes Educator,Nursing Service Providers -163WD1100X,Registered Nurse Dialysis Peritoneal,Nursing Service Providers +163WD1100X,"Registered Nurse Dialysis, Peritoneal",Nursing Service Providers 163WE0003X,Registered Nurse Emergency,Nursing Service Providers 163WE0900X,Registered Nurse Enterostomal Therapy,Nursing Service Providers 163WF0300X,Registered Nurse Flight,Nursing Service Providers @@ -126,16 +126,16 @@ Code,Descriptive_Text,Grouping 163WM0705X,Registered Nurse Medical-Surgical,Nursing Service Providers 163WM1400X,Registered Nurse Nurse Massage Therapist (NMT),Nursing Service Providers 163WN0002X,Registered Nurse Neonatal Intensive Care,Nursing Service Providers -163WN0003X,Registered Nurse Neonatal Low-Risk,Nursing Service Providers +163WN0003X,"Registered Nurse Neonatal, Low-Risk",Nursing Service Providers 163WN0300X,Registered Nurse Nephrology,Nursing Service Providers 163WN0800X,Registered Nurse Neuroscience,Nursing Service Providers 163WN1003X,Registered Nurse Nutrition Support,Nursing Service Providers 163WP0000X,Registered Nurse Pain Management,Nursing Service Providers 163WP0200X,Registered Nurse Pediatrics,Nursing Service Providers 163WP0218X,Registered Nurse Pediatric Oncology,Nursing Service Providers -163WP0807X,Registered Nurse Psych/Mental Health Child & Adolescent,Nursing Service Providers +163WP0807X,"Registered Nurse Psych/Mental Health, Child & Adolescent",Nursing Service Providers 163WP0808X,Registered Nurse Psych/Mental Health,Nursing Service Providers -163WP0809X,Registered Nurse Psych/Mental Health Adult,Nursing Service Providers +163WP0809X,"Registered Nurse Psych/Mental Health, Adult",Nursing Service Providers 163WP1700X,Registered Nurse Perinatal,Nursing Service Providers 163WP2201X,Registered Nurse Ambulatory Care,Nursing Service Providers 163WR0006X,Registered Nurse Registered Nurse First Assistant,Nursing Service Providers @@ -145,9 +145,9 @@ Code,Descriptive_Text,Grouping 163WS0200X,Registered Nurse School,Nursing Service Providers 163WU0100X,Registered Nurse Urology,Nursing Service Providers 163WW0000X,Registered Nurse Wound Care,Nursing Service Providers -163WW0101X,Registered Nurse Women's Health Care Ambulatory,Nursing Service Providers -163WX0002X,Registered Nurse Obstetric High-Risk,Nursing Service Providers -163WX0003X,Registered Nurse Obstetric Inpatient,Nursing Service Providers +163WW0101X,"Registered Nurse Women's Health Care, Ambulatory",Nursing Service Providers +163WX0002X,"Registered Nurse Obstetric, High-Risk",Nursing Service Providers +163WX0003X,"Registered Nurse Obstetric, Inpatient",Nursing Service Providers 163WX0106X,Registered Nurse Occupational Health,Nursing Service Providers 163WX0200X,Registered Nurse Oncology,Nursing Service Providers 163WX0601X,Registered Nurse Otorhinolaryngology & Head-Neck,Nursing Service Providers @@ -157,8 +157,8 @@ Code,Descriptive_Text,Grouping 164W00000X,Licensed Practical Nurse ,Nursing Service Providers 164X00000X,Licensed Vocational Nurse ,Nursing Service Providers 167G00000X,Licensed Psychiatric Technician ,Nursing Service Providers -170100000X,Medical Genetics Ph.D. Medical Genetics ,Other Service Providers -170300000X,Genetic Counselor MS ,Other Service Providers +170100000X,"Medical Genetics, Ph.D. Medical Genetics ",Other Service Providers +170300000X,"Genetic Counselor, MS ",Other Service Providers 171000000X,Military Health Care Provider ,Other Service Providers 1710I1002X,Military Health Care Provider Independent Duty Corpsman,Other Service Providers 1710I1003X,Military Health Care Provider Independent Duty Medical Technicians,Other Service Providers @@ -174,7 +174,7 @@ Code,Descriptive_Text,Grouping 172V00000X,Community Health Worker ,Other Service Providers 173000000X,Legal Medicine ,Other Service Providers 173C00000X,Reflexologist ,Other Service Providers -173F00000X,Sleep Specialist PhD ,Other Service Providers +173F00000X,"Sleep Specialist, PhD ",Other Service Providers 174200000X,Meals ,Other Service Providers 174400000X,Specialist ,Other Service Providers 1744G0900X,Specialist Graphics Designer,Other Service Providers @@ -184,11 +184,11 @@ Code,Descriptive_Text,Grouping 174H00000X,Health Educator ,Other Service Providers 174M00000X,Veterinarian ,Other Service Providers 174MM1900X,Veterinarian Medical Research,Other Service Providers -174N00000X,Lactation Consultant Non-RN ,Other Service Providers +174N00000X,"Lactation Consultant, Non-RN ",Other Service Providers 174V00000X,Clinical Ethicist ,Other Service Providers 175F00000X,Naturopath ,Other Service Providers 175L00000X,Homeopath ,Other Service Providers -175M00000X,Midwife Lay ,Other Service Providers +175M00000X,"Midwife, Lay ",Other Service Providers 175T00000X,Peer Specialist ,Other Service Providers 176B00000X,Midwife ,Other Service Providers 176P00000X,Funeral Director ,Other Service Providers @@ -210,7 +210,7 @@ Code,Descriptive_Text,Grouping 193400000X,Single Specialty ,Group 202C00000X,Independent Medical Examiner ,Allopathic & Osteopathic Physicians 202K00000X,Phlebology ,Allopathic & Osteopathic Physicians -204C00000X,Neuromusculoskeletal Medicine Sports Medicine ,Allopathic & Osteopathic Physicians +204C00000X,"Neuromusculoskeletal Medicine, Sports Medicine ",Allopathic & Osteopathic Physicians 204D00000X,Neuromusculoskeletal Medicine & OMM ,Allopathic & Osteopathic Physicians 204E00000X,Oral & Maxillofacial Surgery ,Allopathic & Osteopathic Physicians 204F00000X,Transplant Surgery ,Allopathic & Osteopathic Physicians @@ -255,7 +255,7 @@ Code,Descriptive_Text,Grouping 207RC0000X,Internal Medicine Cardiovascular Disease,Allopathic & Osteopathic Physicians 207RC0001X,Internal Medicine Clinical Cardiac Electrophysiology,Allopathic & Osteopathic Physicians 207RC0200X,Internal Medicine Critical Care Medicine,Allopathic & Osteopathic Physicians -207RE0101X,Internal Medicine Endocrinology Diabetes & Metabolism,Allopathic & Osteopathic Physicians +207RE0101X,"Internal Medicine Endocrinology, Diabetes & Metabolism",Allopathic & Osteopathic Physicians 207RG0100X,Internal Medicine Gastroenterology,Allopathic & Osteopathic Physicians 207RG0300X,Internal Medicine Geriatric Medicine,Allopathic & Osteopathic Physicians 207RH0000X,Internal Medicine Hematology,Allopathic & Osteopathic Physicians @@ -341,7 +341,7 @@ Code,Descriptive_Text,Grouping 2080H0002X,Pediatrics Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians 2080I0007X,Pediatrics Clinical & Laboratory Immunology,Allopathic & Osteopathic Physicians 2080N0001X,Pediatrics Neonatal-Perinatal Medicine,Allopathic & Osteopathic Physicians -2080P0006X,Pediatrics Developmental – Behavioral Pediatrics,Allopathic & Osteopathic Physicians +2080P0006X,Pediatrics Developmental - Behavioral Pediatrics,Allopathic & Osteopathic Physicians 2080P0008X,Pediatrics Neurodevelopmental Disabilities,Allopathic & Osteopathic Physicians 2080P0201X,Pediatrics Pediatric Allergy/Immunology,Allopathic & Osteopathic Physicians 2080P0202X,Pediatrics Pediatric Cardiology,Allopathic & Osteopathic Physicians @@ -432,7 +432,7 @@ Code,Descriptive_Text,Grouping 208VP0000X,Pain Medicine Pain Medicine,Allopathic & Osteopathic Physicians 208VP0014X,Pain Medicine Interventional Pain Medicine,Allopathic & Osteopathic Physicians 209800000X,Legal Medicine ,Allopathic & Osteopathic Physicians -211D00000X,Assistant Podiatric ,Podiatric Medicine & Surgery Service Providers +211D00000X,"Assistant, Podiatric ",Podiatric Medicine & Surgery Service Providers 213E00000X,Podiatrist ,Podiatric Medicine & Surgery Service Providers 213EG0000X,Podiatrist General Practice,Podiatric Medicine & Surgery Service Providers 213EP0504X,Podiatrist Public Medicine,Podiatric Medicine & Surgery Service Providers @@ -441,168 +441,168 @@ Code,Descriptive_Text,Grouping 213ES0000X,Podiatrist Sports Medicine,Podiatric Medicine & Surgery Service Providers 213ES0103X,Podiatrist Foot & Ankle Surgery,Podiatric Medicine & Surgery Service Providers 213ES0131X,Podiatrist Foot Surgery,Podiatric Medicine & Surgery Service Providers -221700000X,Art Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -222Q00000X,Developmental Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -222Z00000X,Orthotist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -224900000X,Mastectomy Fitter ,Respiratory Developmental Rehabilitative and Restorative Service Providers -224L00000X,Pedorthist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -224P00000X,Prosthetist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -224Y00000X,Clinical Exercise Physiologist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -224Z00000X,Occupational Therapy Assistant ,Respiratory Developmental Rehabilitative and Restorative Service Providers -224ZE0001X,Occupational Therapy Assistant Environmental Modification,Respiratory Developmental Rehabilitative and Restorative Service Providers -224ZF0002X,Occupational Therapy Assistant Feeding Eating & Swallowing,Respiratory Developmental Rehabilitative and Restorative Service Providers -224ZL0004X,Occupational Therapy Assistant Low Vision,Respiratory Developmental Rehabilitative and Restorative Service Providers -224ZR0403X,Occupational Therapy Assistant Driving and Community Mobility,Respiratory Developmental Rehabilitative and Restorative Service Providers -225000000X,Orthotic Fitter ,Respiratory Developmental Rehabilitative and Restorative Service Providers -225100000X,Physical Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -2251C2600X,Physical Therapist Cardiopulmonary,Respiratory Developmental Rehabilitative and Restorative Service Providers -2251E1200X,Physical Therapist Ergonomics,Respiratory Developmental Rehabilitative and Restorative Service Providers -2251E1300X,Physical Therapist Electrophysiology Clinical,Respiratory Developmental Rehabilitative and Restorative Service Providers -2251G0304X,Physical Therapist Geriatrics,Respiratory Developmental Rehabilitative and Restorative Service Providers -2251H1200X,Physical Therapist Hand,Respiratory Developmental Rehabilitative and Restorative Service Providers -2251H1300X,Physical Therapist Human Factors,Respiratory Developmental Rehabilitative and Restorative Service Providers -2251N0400X,Physical Therapist Neurology,Respiratory Developmental Rehabilitative and Restorative Service Providers -2251P0200X,Physical Therapist Pediatrics,Respiratory Developmental Rehabilitative and Restorative Service Providers -2251S0007X,Physical Therapist Sports,Respiratory Developmental Rehabilitative and Restorative Service Providers -2251X0800X,Physical Therapist Orthopedic,Respiratory Developmental Rehabilitative and Restorative Service Providers -225200000X,Physical Therapy Assistant ,Respiratory Developmental Rehabilitative and Restorative Service Providers -225400000X,Rehabilitation Practitioner ,Respiratory Developmental Rehabilitative and Restorative Service Providers -225500000X,Specialist/Technologist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -2255A2300X,Specialist/Technologist Athletic Trainer,Respiratory Developmental Rehabilitative and Restorative Service Providers -2255R0406X,Specialist/Technologist Rehabilitation Blind,Respiratory Developmental Rehabilitative and Restorative Service Providers -225600000X,Dance Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -225700000X,Massage Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -225800000X,Recreation Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -225A00000X,Music Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -225B00000X,Pulmonary Function Technologist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -225C00000X,Rehabilitation Counselor ,Respiratory Developmental Rehabilitative and Restorative Service Providers -225CA2400X,Rehabilitation Counselor Assistive Technology Practitioner,Respiratory Developmental Rehabilitative and Restorative Service Providers -225CA2500X,Rehabilitation Counselor Assistive Technology Supplier,Respiratory Developmental Rehabilitative and Restorative Service Providers -225CX0006X,Rehabilitation Counselor Orientation and Mobility Training Provider,Respiratory Developmental Rehabilitative and Restorative Service Providers -225X00000X,Occupational Therapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -225XE0001X,Occupational Therapist Environmental Modification,Respiratory Developmental Rehabilitative and Restorative Service Providers -225XE1200X,Occupational Therapist Ergonomics,Respiratory Developmental Rehabilitative and Restorative Service Providers -225XF0002X,Occupational Therapist Feeding Eating & Swallowing,Respiratory Developmental Rehabilitative and Restorative Service Providers -225XG0600X,Occupational Therapist Gerontology,Respiratory Developmental Rehabilitative and Restorative Service Providers -225XH1200X,Occupational Therapist Hand,Respiratory Developmental Rehabilitative and Restorative Service Providers -225XH1300X,Occupational Therapist Human Factors,Respiratory Developmental Rehabilitative and Restorative Service Providers -225XL0004X,Occupational Therapist Low Vision,Respiratory Developmental Rehabilitative and Restorative Service Providers -225XM0800X,Occupational Therapist Mental Health,Respiratory Developmental Rehabilitative and Restorative Service Providers -225XN1300X,Occupational Therapist Neurorehabilitation,Respiratory Developmental Rehabilitative and Restorative Service Providers -225XP0019X,Occupational Therapist Physical Rehabilitation,Respiratory Developmental Rehabilitative and Restorative Service Providers -225XP0200X,Occupational Therapist Pediatrics,Respiratory Developmental Rehabilitative and Restorative Service Providers -225XR0403X,Occupational Therapist Driving and Community Mobility,Respiratory Developmental Rehabilitative and Restorative Service Providers -226000000X,Recreational Therapist Assistant ,Respiratory Developmental Rehabilitative and Restorative Service Providers -226300000X,Kinesiotherapist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -227800000X,Respiratory Therapist Certified ,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278C0205X,Respiratory Therapist Certified Critical Care,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278E0002X,Respiratory Therapist Certified Emergency Care,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278E1000X,Respiratory Therapist Certified Educational,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278G0305X,Respiratory Therapist Certified Geriatric Care,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278G1100X,Respiratory Therapist Certified General Care,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278H0200X,Respiratory Therapist Certified Home Health,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278P1004X,Respiratory Therapist Certified Pulmonary Diagnostics,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278P1005X,Respiratory Therapist Certified Pulmonary Rehabilitation,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278P1006X,Respiratory Therapist Certified Pulmonary Function Technologist,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278P3800X,Respiratory Therapist Certified Palliative/Hospice,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278P3900X,Respiratory Therapist Certified Neonatal/Pediatrics,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278P4000X,Respiratory Therapist Certified Patient Transport,Respiratory Developmental Rehabilitative and Restorative Service Providers -2278S1500X,Respiratory Therapist Certified SNF/Subacute Care,Respiratory Developmental Rehabilitative and Restorative Service Providers -227900000X,Respiratory Therapist Registered ,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279C0205X,Respiratory Therapist Registered Critical Care,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279E0002X,Respiratory Therapist Registered Emergency Care,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279E1000X,Respiratory Therapist Registered Educational,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279G0305X,Respiratory Therapist Registered Geriatric Care,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279G1100X,Respiratory Therapist Registered General Care,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279H0200X,Respiratory Therapist Registered Home Health,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279P1004X,Respiratory Therapist Registered Pulmonary Diagnostics,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279P1005X,Respiratory Therapist Registered Pulmonary Rehabilitation,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279P1006X,Respiratory Therapist Registered Pulmonary Function Technologist,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279P3800X,Respiratory Therapist Registered Palliative/Hospice,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279P3900X,Respiratory Therapist Registered Neonatal/Pediatrics,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279P4000X,Respiratory Therapist Registered Patient Transport,Respiratory Developmental Rehabilitative and Restorative Service Providers -2279S1500X,Respiratory Therapist Registered SNF/Subacute Care,Respiratory Developmental Rehabilitative and Restorative Service Providers -229N00000X,Anaplastologist ,Respiratory Developmental Rehabilitative and Restorative Service Providers -231H00000X,Audiologist ,Speech Language and Hearing Service Providers -231HA2400X,Audiologist Assistive Technology Practitioner,Speech Language and Hearing Service Providers -231HA2500X,Audiologist Assistive Technology Supplier,Speech Language and Hearing Service Providers -235500000X,Specialist/Technologist ,Speech Language and Hearing Service Providers -2355A2700X,Specialist/Technologist Audiology Assistant,Speech Language and Hearing Service Providers -2355S0801X,Specialist/Technologist Speech-Language Assistant,Speech Language and Hearing Service Providers -235Z00000X,Speech-Language Pathologist ,Speech Language and Hearing Service Providers -237600000X,Audiologist-Hearing Aid Fitter ,Speech Language and Hearing Service Providers -237700000X,Hearing Instrument Specialist ,Speech Language and Hearing Service Providers -242T00000X,Perfusionist ,Technologists Technicians & Other Technical Service Providers -243U00000X,Radiology Practitioner Assistant ,Technologists Technicians & Other Technical Service Providers -246Q00000X,Spec/Tech Pathology ,Technologists Technicians & Other Technical Service Providers -246QB0000X,Spec/Tech Pathology Blood Banking,Technologists Technicians & Other Technical Service Providers -246QC1000X,Spec/Tech Pathology Chemistry,Technologists Technicians & Other Technical Service Providers -246QC2700X,Spec/Tech Pathology Cytotechnology,Technologists Technicians & Other Technical Service Providers -246QH0000X,Spec/Tech Pathology Hematology,Technologists Technicians & Other Technical Service Providers -246QH0401X,Spec/Tech Pathology Hemapheresis Practitioner,Technologists Technicians & Other Technical Service Providers -246QH0600X,Spec/Tech Pathology Histology,Technologists Technicians & Other Technical Service Providers -246QI0000X,Spec/Tech Pathology Immunology,Technologists Technicians & Other Technical Service Providers -246QL0900X,Spec/Tech Pathology Laboratory Management,Technologists Technicians & Other Technical Service Providers -246QL0901X,Spec/Tech Pathology Laboratory Management Diplomate,Technologists Technicians & Other Technical Service Providers -246QM0706X,Spec/Tech Pathology Medical Technologist,Technologists Technicians & Other Technical Service Providers -246QM0900X,Spec/Tech Pathology Microbiology,Technologists Technicians & Other Technical Service Providers -246R00000X,Technician Pathology ,Technologists Technicians & Other Technical Service Providers -246RH0600X,Technician Pathology Histology,Technologists Technicians & Other Technical Service Providers -246RM2200X,Technician Pathology Medical Laboratory,Technologists Technicians & Other Technical Service Providers -246RP1900X,Technician Pathology Phlebotomy,Technologists Technicians & Other Technical Service Providers -246W00000X,Technician Cardiology ,Technologists Technicians & Other Technical Service Providers -246X00000X,Spec/Tech Cardiovascular ,Technologists Technicians & Other Technical Service Providers -246XC2901X,Spec/Tech Cardiovascular Cardiovascular Invasive Specialist,Technologists Technicians & Other Technical Service Providers -246XC2903X,Spec/Tech Cardiovascular Vascular Specialist,Technologists Technicians & Other Technical Service Providers -246XS1301X,Spec/Tech Cardiovascular Sonography,Technologists Technicians & Other Technical Service Providers -246Y00000X,Spec/Tech Health Info ,Technologists Technicians & Other Technical Service Providers -246YC3301X,Spec/Tech Health Info Coding Specialist Hospital Based,Technologists Technicians & Other Technical Service Providers -246YC3302X,Spec/Tech Health Info Coding Specialist Physician Office Based,Technologists Technicians & Other Technical Service Providers -246YR1600X,Spec/Tech Health Info Registered Record Administrator,Technologists Technicians & Other Technical Service Providers -246Z00000X,Specialist/Technologist Other ,Technologists Technicians & Other Technical Service Providers -246ZA2600X,Specialist/Technologist Other Art Medical,Technologists Technicians & Other Technical Service Providers -246ZB0301X,Specialist/Technologist Other Biomedical Engineering,Technologists Technicians & Other Technical Service Providers -246ZB0302X,Specialist/Technologist Other Biomedical Photographer,Technologists Technicians & Other Technical Service Providers -246ZB0500X,Specialist/Technologist Other Biochemist,Technologists Technicians & Other Technical Service Providers -246ZB0600X,Specialist/Technologist Other Biostatistician,Technologists Technicians & Other Technical Service Providers -246ZC0007X,Specialist/Technologist Other Surgical Assistant,Technologists Technicians & Other Technical Service Providers -246ZE0500X,Specialist/Technologist Other EEG,Technologists Technicians & Other Technical Service Providers -246ZE0600X,Specialist/Technologist Other Electroneurodiagnostic,Technologists Technicians & Other Technical Service Providers -246ZG0701X,Specialist/Technologist Other Graphics Methods,Technologists Technicians & Other Technical Service Providers -246ZG1000X,Specialist/Technologist Other Geneticist Medical (PhD),Technologists Technicians & Other Technical Service Providers -246ZI1000X,Specialist/Technologist Other Illustration Medical,Technologists Technicians & Other Technical Service Providers -246ZN0300X,Specialist/Technologist Other Nephrology,Technologists Technicians & Other Technical Service Providers -246ZS0410X,Specialist/Technologist Other Surgical Technologist,Technologists Technicians & Other Technical Service Providers -246ZX2200X,Specialist/Technologist Other Orthopedic Assistant,Technologists Technicians & Other Technical Service Providers -247000000X,Technician Health Information ,Technologists Technicians & Other Technical Service Providers -2470A2800X,Technician Health Information Assistant Record Technician,Technologists Technicians & Other Technical Service Providers -247100000X,Radiologic Technologist ,Technologists Technicians & Other Technical Service Providers -2471B0102X,Radiologic Technologist Bone Densitometry,Technologists Technicians & Other Technical Service Providers -2471C1101X,Radiologic Technologist Cardiovascular-Interventional Technology,Technologists Technicians & Other Technical Service Providers -2471C1106X,Radiologic Technologist Cardiac-Interventional Technology,Technologists Technicians & Other Technical Service Providers -2471C3401X,Radiologic Technologist Computed Tomography,Technologists Technicians & Other Technical Service Providers -2471C3402X,Radiologic Technologist Radiography,Technologists Technicians & Other Technical Service Providers -2471M1202X,Radiologic Technologist Magnetic Resonance Imaging,Technologists Technicians & Other Technical Service Providers -2471M2300X,Radiologic Technologist Mammography,Technologists Technicians & Other Technical Service Providers -2471N0900X,Radiologic Technologist Nuclear Medicine Technology,Technologists Technicians & Other Technical Service Providers -2471Q0001X,Radiologic Technologist Quality Management,Technologists Technicians & Other Technical Service Providers -2471R0002X,Radiologic Technologist Radiation Therapy,Technologists Technicians & Other Technical Service Providers -2471S1302X,Radiologic Technologist Sonography,Technologists Technicians & Other Technical Service Providers -2471V0105X,Radiologic Technologist Vascular Sonography,Technologists Technicians & Other Technical Service Providers -2471V0106X,Radiologic Technologist Vascular-Interventional Technology,Technologists Technicians & Other Technical Service Providers -247200000X,Technician Other ,Technologists Technicians & Other Technical Service Providers -2472B0301X,Technician Other Biomedical Engineering,Technologists Technicians & Other Technical Service Providers -2472D0500X,Technician Other Darkroom,Technologists Technicians & Other Technical Service Providers -2472E0500X,Technician Other EEG,Technologists Technicians & Other Technical Service Providers -2472R0900X,Technician Other Renal Dialysis,Technologists Technicians & Other Technical Service Providers -2472V0600X,Technician Other Veterinary,Technologists Technicians & Other Technical Service Providers -247ZC0005X,Pathology Clinical Laboratory Director Non-physician,Technologists Technicians & Other Technical Service Providers +221700000X,Art Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +222Q00000X,Developmental Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +222Z00000X,Orthotist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224900000X,Mastectomy Fitter ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224L00000X,Pedorthist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224P00000X,Prosthetist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224Y00000X,Clinical Exercise Physiologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224Z00000X,Occupational Therapy Assistant ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224ZE0001X,Occupational Therapy Assistant Environmental Modification,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224ZF0002X,"Occupational Therapy Assistant Feeding, Eating & Swallowing","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224ZL0004X,Occupational Therapy Assistant Low Vision,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224ZR0403X,Occupational Therapy Assistant Driving and Community Mobility,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225000000X,Orthotic Fitter ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225100000X,Physical Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251C2600X,Physical Therapist Cardiopulmonary,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251E1200X,Physical Therapist Ergonomics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251E1300X,"Physical Therapist Electrophysiology, Clinical","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251G0304X,Physical Therapist Geriatrics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251H1200X,Physical Therapist Hand,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251H1300X,Physical Therapist Human Factors,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251N0400X,Physical Therapist Neurology,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251P0200X,Physical Therapist Pediatrics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251S0007X,Physical Therapist Sports,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251X0800X,Physical Therapist Orthopedic,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225200000X,Physical Therapy Assistant ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225400000X,Rehabilitation Practitioner ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225500000X,Specialist/Technologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2255A2300X,Specialist/Technologist Athletic Trainer,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2255R0406X,"Specialist/Technologist Rehabilitation, Blind","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225600000X,Dance Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225700000X,Massage Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225800000X,Recreation Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225A00000X,Music Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225B00000X,Pulmonary Function Technologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225C00000X,Rehabilitation Counselor ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225CA2400X,Rehabilitation Counselor Assistive Technology Practitioner,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225CA2500X,Rehabilitation Counselor Assistive Technology Supplier,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225CX0006X,Rehabilitation Counselor Orientation and Mobility Training Provider,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225X00000X,Occupational Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XE0001X,Occupational Therapist Environmental Modification,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XE1200X,Occupational Therapist Ergonomics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XF0002X,"Occupational Therapist Feeding, Eating & Swallowing","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XG0600X,Occupational Therapist Gerontology,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XH1200X,Occupational Therapist Hand,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XH1300X,Occupational Therapist Human Factors,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XL0004X,Occupational Therapist Low Vision,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XM0800X,Occupational Therapist Mental Health,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XN1300X,Occupational Therapist Neurorehabilitation,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XP0019X,Occupational Therapist Physical Rehabilitation,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XP0200X,Occupational Therapist Pediatrics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XR0403X,Occupational Therapist Driving and Community Mobility,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +226000000X,Recreational Therapist Assistant ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +226300000X,Kinesiotherapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +227800000X,"Respiratory Therapist, Certified ","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278C0205X,"Respiratory Therapist, Certified Critical Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278E0002X,"Respiratory Therapist, Certified Emergency Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278E1000X,"Respiratory Therapist, Certified Educational","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278G0305X,"Respiratory Therapist, Certified Geriatric Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278G1100X,"Respiratory Therapist, Certified General Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278H0200X,"Respiratory Therapist, Certified Home Health","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P1004X,"Respiratory Therapist, Certified Pulmonary Diagnostics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P1005X,"Respiratory Therapist, Certified Pulmonary Rehabilitation","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P1006X,"Respiratory Therapist, Certified Pulmonary Function Technologist","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P3800X,"Respiratory Therapist, Certified Palliative/Hospice","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P3900X,"Respiratory Therapist, Certified Neonatal/Pediatrics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P4000X,"Respiratory Therapist, Certified Patient Transport","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278S1500X,"Respiratory Therapist, Certified SNF/Subacute Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +227900000X,"Respiratory Therapist, Registered ","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279C0205X,"Respiratory Therapist, Registered Critical Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279E0002X,"Respiratory Therapist, Registered Emergency Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279E1000X,"Respiratory Therapist, Registered Educational","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279G0305X,"Respiratory Therapist, Registered Geriatric Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279G1100X,"Respiratory Therapist, Registered General Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279H0200X,"Respiratory Therapist, Registered Home Health","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P1004X,"Respiratory Therapist, Registered Pulmonary Diagnostics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P1005X,"Respiratory Therapist, Registered Pulmonary Rehabilitation","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P1006X,"Respiratory Therapist, Registered Pulmonary Function Technologist","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P3800X,"Respiratory Therapist, Registered Palliative/Hospice","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P3900X,"Respiratory Therapist, Registered Neonatal/Pediatrics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P4000X,"Respiratory Therapist, Registered Patient Transport","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279S1500X,"Respiratory Therapist, Registered SNF/Subacute Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +229N00000X,Anaplastologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +231H00000X,Audiologist ,"Speech, Language and Hearing Service Providers" +231HA2400X,Audiologist Assistive Technology Practitioner,"Speech, Language and Hearing Service Providers" +231HA2500X,Audiologist Assistive Technology Supplier,"Speech, Language and Hearing Service Providers" +235500000X,Specialist/Technologist ,"Speech, Language and Hearing Service Providers" +2355A2700X,Specialist/Technologist Audiology Assistant,"Speech, Language and Hearing Service Providers" +2355S0801X,Specialist/Technologist Speech-Language Assistant,"Speech, Language and Hearing Service Providers" +235Z00000X,Speech-Language Pathologist ,"Speech, Language and Hearing Service Providers" +237600000X,Audiologist-Hearing Aid Fitter ,"Speech, Language and Hearing Service Providers" +237700000X,Hearing Instrument Specialist ,"Speech, Language and Hearing Service Providers" +242T00000X,Perfusionist ,"Technologists, Technicians & Other Technical Service Providers" +243U00000X,Radiology Practitioner Assistant ,"Technologists, Technicians & Other Technical Service Providers" +246Q00000X,"Spec/Tech, Pathology ","Technologists, Technicians & Other Technical Service Providers" +246QB0000X,"Spec/Tech, Pathology Blood Banking","Technologists, Technicians & Other Technical Service Providers" +246QC1000X,"Spec/Tech, Pathology Chemistry","Technologists, Technicians & Other Technical Service Providers" +246QC2700X,"Spec/Tech, Pathology Cytotechnology","Technologists, Technicians & Other Technical Service Providers" +246QH0000X,"Spec/Tech, Pathology Hematology","Technologists, Technicians & Other Technical Service Providers" +246QH0401X,"Spec/Tech, Pathology Hemapheresis Practitioner","Technologists, Technicians & Other Technical Service Providers" +246QH0600X,"Spec/Tech, Pathology Histology","Technologists, Technicians & Other Technical Service Providers" +246QI0000X,"Spec/Tech, Pathology Immunology","Technologists, Technicians & Other Technical Service Providers" +246QL0900X,"Spec/Tech, Pathology Laboratory Management","Technologists, Technicians & Other Technical Service Providers" +246QL0901X,"Spec/Tech, Pathology Laboratory Management, Diplomate","Technologists, Technicians & Other Technical Service Providers" +246QM0706X,"Spec/Tech, Pathology Medical Technologist","Technologists, Technicians & Other Technical Service Providers" +246QM0900X,"Spec/Tech, Pathology Microbiology","Technologists, Technicians & Other Technical Service Providers" +246R00000X,"Technician, Pathology ","Technologists, Technicians & Other Technical Service Providers" +246RH0600X,"Technician, Pathology Histology","Technologists, Technicians & Other Technical Service Providers" +246RM2200X,"Technician, Pathology Medical Laboratory","Technologists, Technicians & Other Technical Service Providers" +246RP1900X,"Technician, Pathology Phlebotomy","Technologists, Technicians & Other Technical Service Providers" +246W00000X,"Technician, Cardiology ","Technologists, Technicians & Other Technical Service Providers" +246X00000X,"Spec/Tech, Cardiovascular ","Technologists, Technicians & Other Technical Service Providers" +246XC2901X,"Spec/Tech, Cardiovascular Cardiovascular Invasive Specialist","Technologists, Technicians & Other Technical Service Providers" +246XC2903X,"Spec/Tech, Cardiovascular Vascular Specialist","Technologists, Technicians & Other Technical Service Providers" +246XS1301X,"Spec/Tech, Cardiovascular Sonography","Technologists, Technicians & Other Technical Service Providers" +246Y00000X,"Spec/Tech, Health Info ","Technologists, Technicians & Other Technical Service Providers" +246YC3301X,"Spec/Tech, Health Info Coding Specialist, Hospital Based","Technologists, Technicians & Other Technical Service Providers" +246YC3302X,"Spec/Tech, Health Info Coding Specialist, Physician Office Based","Technologists, Technicians & Other Technical Service Providers" +246YR1600X,"Spec/Tech, Health Info Registered Record Administrator","Technologists, Technicians & Other Technical Service Providers" +246Z00000X,"Specialist/Technologist, Other ","Technologists, Technicians & Other Technical Service Providers" +246ZA2600X,"Specialist/Technologist, Other Art, Medical","Technologists, Technicians & Other Technical Service Providers" +246ZB0301X,"Specialist/Technologist, Other Biomedical Engineering","Technologists, Technicians & Other Technical Service Providers" +246ZB0302X,"Specialist/Technologist, Other Biomedical Photographer","Technologists, Technicians & Other Technical Service Providers" +246ZB0500X,"Specialist/Technologist, Other Biochemist","Technologists, Technicians & Other Technical Service Providers" +246ZB0600X,"Specialist/Technologist, Other Biostatistician","Technologists, Technicians & Other Technical Service Providers" +246ZC0007X,"Specialist/Technologist, Other Surgical Assistant","Technologists, Technicians & Other Technical Service Providers" +246ZE0500X,"Specialist/Technologist, Other EEG","Technologists, Technicians & Other Technical Service Providers" +246ZE0600X,"Specialist/Technologist, Other Electroneurodiagnostic","Technologists, Technicians & Other Technical Service Providers" +246ZG0701X,"Specialist/Technologist, Other Graphics Methods","Technologists, Technicians & Other Technical Service Providers" +246ZG1000X,"Specialist/Technologist, Other Geneticist, Medical (PhD)","Technologists, Technicians & Other Technical Service Providers" +246ZI1000X,"Specialist/Technologist, Other Illustration, Medical","Technologists, Technicians & Other Technical Service Providers" +246ZN0300X,"Specialist/Technologist, Other Nephrology","Technologists, Technicians & Other Technical Service Providers" +246ZS0410X,"Specialist/Technologist, Other Surgical Technologist","Technologists, Technicians & Other Technical Service Providers" +246ZX2200X,"Specialist/Technologist, Other Orthopedic Assistant","Technologists, Technicians & Other Technical Service Providers" +247000000X,"Technician, Health Information ","Technologists, Technicians & Other Technical Service Providers" +2470A2800X,"Technician, Health Information Assistant Record Technician","Technologists, Technicians & Other Technical Service Providers" +247100000X,Radiologic Technologist ,"Technologists, Technicians & Other Technical Service Providers" +2471B0102X,Radiologic Technologist Bone Densitometry,"Technologists, Technicians & Other Technical Service Providers" +2471C1101X,Radiologic Technologist Cardiovascular-Interventional Technology,"Technologists, Technicians & Other Technical Service Providers" +2471C1106X,Radiologic Technologist Cardiac-Interventional Technology,"Technologists, Technicians & Other Technical Service Providers" +2471C3401X,Radiologic Technologist Computed Tomography,"Technologists, Technicians & Other Technical Service Providers" +2471C3402X,Radiologic Technologist Radiography,"Technologists, Technicians & Other Technical Service Providers" +2471M1202X,Radiologic Technologist Magnetic Resonance Imaging,"Technologists, Technicians & Other Technical Service Providers" +2471M2300X,Radiologic Technologist Mammography,"Technologists, Technicians & Other Technical Service Providers" +2471N0900X,Radiologic Technologist Nuclear Medicine Technology,"Technologists, Technicians & Other Technical Service Providers" +2471Q0001X,Radiologic Technologist Quality Management,"Technologists, Technicians & Other Technical Service Providers" +2471R0002X,Radiologic Technologist Radiation Therapy,"Technologists, Technicians & Other Technical Service Providers" +2471S1302X,Radiologic Technologist Sonography,"Technologists, Technicians & Other Technical Service Providers" +2471V0105X,Radiologic Technologist Vascular Sonography,"Technologists, Technicians & Other Technical Service Providers" +2471V0106X,Radiologic Technologist Vascular-Interventional Technology,"Technologists, Technicians & Other Technical Service Providers" +247200000X,"Technician, Other ","Technologists, Technicians & Other Technical Service Providers" +2472B0301X,"Technician, Other Biomedical Engineering","Technologists, Technicians & Other Technical Service Providers" +2472D0500X,"Technician, Other Darkroom","Technologists, Technicians & Other Technical Service Providers" +2472E0500X,"Technician, Other EEG","Technologists, Technicians & Other Technical Service Providers" +2472R0900X,"Technician, Other Renal Dialysis","Technologists, Technicians & Other Technical Service Providers" +2472V0600X,"Technician, Other Veterinary","Technologists, Technicians & Other Technical Service Providers" +247ZC0005X,"Pathology Clinical Laboratory Director, Non-physician","Technologists, Technicians & Other Technical Service Providers" 251300000X,Local Education Agency (LEA) ,Agencies 251B00000X,Case Management ,Agencies -251C00000X,Day Training Developmentally Disabled Services ,Agencies +251C00000X,"Day Training, Developmentally Disabled Services ",Agencies 251E00000X,Home Health ,Agencies 251F00000X,Home Infusion ,Agencies -251G00000X,Hospice Care Community Based ,Agencies +251G00000X,"Hospice Care, Community Based ",Agencies 251J00000X,Nursing Care ,Agencies 251K00000X,Public Health or Welfare ,Agencies 251S00000X,Community/Behavioral Health ,Agencies @@ -628,7 +628,7 @@ Code,Descriptive_Text,Grouping 261QE0002X,Clinic/Center Emergency Care,Ambulatory Health Care Facilities 261QE0700X,Clinic/Center End-Stage Renal Disease (ESRD) Treatment,Ambulatory Health Care Facilities 261QE0800X,Clinic/Center Endoscopy,Ambulatory Health Care Facilities -261QF0050X,Clinic/Center Family Planning Non-Surgical,Ambulatory Health Care Facilities +261QF0050X,"Clinic/Center Family Planning, Non-Surgical",Ambulatory Health Care Facilities 261QF0400X,Clinic/Center Federally Qualified Health Center (FQHC),Ambulatory Health Care Facilities 261QG0250X,Clinic/Center Genetics,Ambulatory Health Care Facilities 261QH0100X,Clinic/Center Health Service,Ambulatory Health Care Facilities @@ -648,21 +648,21 @@ Code,Descriptive_Text,Grouping 261QM2500X,Clinic/Center Medical Specialty,Ambulatory Health Care Facilities 261QM2800X,Clinic/Center Methadone Clinic,Ambulatory Health Care Facilities 261QM3000X,Clinic/Center Medically Fragile Intants and Children Day Care,Ambulatory Health Care Facilities -261QP0904X,Clinic/Center Public Health Federal,Ambulatory Health Care Facilities -261QP0905X,Clinic/Center Public Health State or Local,Ambulatory Health Care Facilities +261QP0904X,"Clinic/Center Public Health, Federal",Ambulatory Health Care Facilities +261QP0905X,"Clinic/Center Public Health, State or Local",Ambulatory Health Care Facilities 261QP1100X,Clinic/Center Podiatric,Ambulatory Health Care Facilities 261QP2000X,Clinic/Center Physical Therapy,Ambulatory Health Care Facilities 261QP2300X,Clinic/Center Primary Care,Ambulatory Health Care Facilities 261QP2400X,Clinic/Center Prison Health,Ambulatory Health Care Facilities 261QP3300X,Clinic/Center Pain,Ambulatory Health Care Facilities 261QR0200X,Clinic/Center Radiology,Ambulatory Health Care Facilities -261QR0206X,Clinic/Center Radiology Mammography,Ambulatory Health Care Facilities -261QR0207X,Clinic/Center Radiology Mobile Mammography,Ambulatory Health Care Facilities -261QR0208X,Clinic/Center Radiology Mobile,Ambulatory Health Care Facilities +261QR0206X,"Clinic/Center Radiology, Mammography",Ambulatory Health Care Facilities +261QR0207X,"Clinic/Center Radiology, Mobile Mammography",Ambulatory Health Care Facilities +261QR0208X,"Clinic/Center Radiology, Mobile",Ambulatory Health Care Facilities 261QR0400X,Clinic/Center Rehabilitation,Ambulatory Health Care Facilities -261QR0401X,Clinic/Center Rehabilitation Comprehensive Outpatient Rehabilitation Facility (CORF),Ambulatory Health Care Facilities -261QR0404X,Clinic/Center Rehabilitation Cardiac Facilities,Ambulatory Health Care Facilities -261QR0405X,Clinic/Center Rehabilitation Substance Use Disorder,Ambulatory Health Care Facilities +261QR0401X,"Clinic/Center Rehabilitation, Comprehensive Outpatient Rehabilitation Facility (CORF)",Ambulatory Health Care Facilities +261QR0404X,"Clinic/Center Rehabilitation, Cardiac Facilities",Ambulatory Health Care Facilities +261QR0405X,"Clinic/Center Rehabilitation, Substance Use Disorder",Ambulatory Health Care Facilities 261QR0800X,Clinic/Center Recovery Care,Ambulatory Health Care Facilities 261QR1100X,Clinic/Center Research,Ambulatory Health Care Facilities 261QR1300X,Clinic/Center Rural Health,Ambulatory Health Care Facilities @@ -674,12 +674,12 @@ Code,Descriptive_Text,Grouping 261QV0200X,Clinic/Center VA,Ambulatory Health Care Facilities 261QX0100X,Clinic/Center Occupational Medicine,Ambulatory Health Care Facilities 261QX0200X,Clinic/Center Oncology,Ambulatory Health Care Facilities -261QX0203X,Clinic/Center Oncology Radiation,Ambulatory Health Care Facilities +261QX0203X,"Clinic/Center Oncology, Radiation",Ambulatory Health Care Facilities 273100000X,Epilepsy Unit ,Hospital Units 273R00000X,Psychiatric Unit ,Hospital Units 273Y00000X,Rehabilitation Unit ,Hospital Units 275N00000X,Medicare Defined Swing Bed Unit ,Hospital Units -276400000X,Rehabilitation Substance Use Disorder Unit ,Hospital Units +276400000X,"Rehabilitation, Substance Use Disorder Unit ",Hospital Units 281P00000X,Chronic Disease Hospital ,Hospitals 281PC2000X,Chronic Disease Hospital Children,Hospitals 282E00000X,Long Term Care Hospital ,Hospitals @@ -707,26 +707,26 @@ Code,Descriptive_Text,Grouping 305R00000X,Preferred Provider Organization ,Managed Care Organizations 305S00000X,Point of Service ,Managed Care Organizations 310400000X,Assisted Living Facility ,Nursing & Custodial Care Facilities -3104A0625X,Assisted Living Facility Assisted Living Mental Illness,Nursing & Custodial Care Facilities -3104A0630X,Assisted Living Facility Assisted Living Behavioral Disturbances,Nursing & Custodial Care Facilities -310500000X,Intermediate Care Facility Mental Illness ,Nursing & Custodial Care Facilities +3104A0625X,"Assisted Living Facility Assisted Living, Mental Illness",Nursing & Custodial Care Facilities +3104A0630X,"Assisted Living Facility Assisted Living, Behavioral Disturbances",Nursing & Custodial Care Facilities +310500000X,"Intermediate Care Facility, Mental Illness ",Nursing & Custodial Care Facilities 311500000X,Alzheimer Center (Dementia Center) ,Nursing & Custodial Care Facilities 311Z00000X,Custodial Care Facility ,Nursing & Custodial Care Facilities 311ZA0620X,Custodial Care Facility Adult Care Home,Nursing & Custodial Care Facilities 313M00000X,Nursing Facility/Intermediate Care Facility ,Nursing & Custodial Care Facilities 314000000X,Skilled Nursing Facility ,Nursing & Custodial Care Facilities -3140N1450X,Skilled Nursing Facility Nursing Care Pediatric,Nursing & Custodial Care Facilities -315D00000X,Hospice Inpatient ,Nursing & Custodial Care Facilities -315P00000X,Intermediate Care Facility Mentally Retarded ,Nursing & Custodial Care Facilities +3140N1450X,"Skilled Nursing Facility Nursing Care, Pediatric",Nursing & Custodial Care Facilities +315D00000X,"Hospice, Inpatient ",Nursing & Custodial Care Facilities +315P00000X,"Intermediate Care Facility, Mentally Retarded ",Nursing & Custodial Care Facilities 317400000X,Christian Science Facility ,Nursing & Custodial Care Facilities -320600000X,Residential Treatment Facility Mental Retardation and/or Developmental Disabilities ,Residential Treatment Facilities -320700000X,Residential Treatment Facility Physical Disabilities ,Residential Treatment Facilities -320800000X,Community Based Residential Treatment Facility Mental Illness ,Residential Treatment Facilities -320900000X,Community Based Residential Treatment Mental Retardation and/or Developmental Disabilities ,Residential Treatment Facilities -322D00000X,Residential Treatment Facility Emotionally Disturbed Children ,Residential Treatment Facilities +320600000X,"Residential Treatment Facility, Mental Retardation and/or Developmental Disabilities ",Residential Treatment Facilities +320700000X,"Residential Treatment Facility, Physical Disabilities ",Residential Treatment Facilities +320800000X,"Community Based Residential Treatment Facility, Mental Illness ",Residential Treatment Facilities +320900000X,"Community Based Residential Treatment, Mental Retardation and/or Developmental Disabilities ",Residential Treatment Facilities +322D00000X,"Residential Treatment Facility, Emotionally Disturbed Children ",Residential Treatment Facilities 323P00000X,Psychiatric Residential Treatment Facility ,Residential Treatment Facilities 324500000X,Substance Abuse Rehabilitation Facility ,Residential Treatment Facilities -3245S0500X,Substance Abuse Rehabilitation Facility Substance Abuse Treatment Children,Residential Treatment Facilities +3245S0500X,"Substance Abuse Rehabilitation Facility Substance Abuse Treatment, Children",Residential Treatment Facilities 331L00000X,Blood Bank ,Suppliers 332000000X,Military/U.S. Coast Guard Pharmacy ,Suppliers 332100000X,Department of Veterans Affairs (VA) Pharmacy ,Suppliers @@ -739,7 +739,7 @@ Code,Descriptive_Text,Grouping 332BP3500X,Durable Medical Equipment & Medical Supplies Parenteral & Enteral Nutrition,Suppliers 332BX2000X,Durable Medical Equipment & Medical Supplies Oxygen Equipment & Supplies,Suppliers 332G00000X,Eye Bank ,Suppliers -332H00000X,Eyewear Supplier (Equipment not the service) ,Suppliers +332H00000X,"Eyewear Supplier (Equipment, not the service) ",Suppliers 332S00000X,Hearing Aid Equipment ,Suppliers 332U00000X,Home Delivered Meals ,Suppliers 333300000X,Emergency Response System Companies ,Suppliers @@ -763,9 +763,9 @@ Code,Descriptive_Text,Grouping 3416L0300X,Ambulance Land Transport,Transportation Services 3416S0300X,Ambulance Water Transport,Transportation Services 341800000X,Military/U.S. Coast Guard Transport ,Transportation Services -3418M1110X,Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance Ground Transport,Transportation Services -3418M1120X,Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance Air Transport,Transportation Services -3418M1130X,Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance Water Transport,Transportation Services +3418M1110X,"Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance, Ground Transport",Transportation Services +3418M1120X,"Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance, Air Transport",Transportation Services +3418M1130X,"Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance, Water Transport",Transportation Services 343800000X,Secured Medical Transport (VAN) ,Transportation Services 343900000X,Non-emergency Medical Transport (VAN) ,Transportation Services 344600000X,Taxi ,Transportation Services @@ -785,9 +785,9 @@ Code,Descriptive_Text,Grouping 363LF0000X,Nurse Practitioner Family,Physician Assistants & Advanced Practice Nursing Providers 363LG0600X,Nurse Practitioner Gerontology,Physician Assistants & Advanced Practice Nursing Providers 363LN0000X,Nurse Practitioner Neonatal,Physician Assistants & Advanced Practice Nursing Providers -363LN0005X,Nurse Practitioner Neonatal Critical Care,Physician Assistants & Advanced Practice Nursing Providers +363LN0005X,"Nurse Practitioner Neonatal, Critical Care",Physician Assistants & Advanced Practice Nursing Providers 363LP0200X,Nurse Practitioner Pediatrics,Physician Assistants & Advanced Practice Nursing Providers -363LP0222X,Nurse Practitioner Pediatrics Critical Care,Physician Assistants & Advanced Practice Nursing Providers +363LP0222X,"Nurse Practitioner Pediatrics, Critical Care",Physician Assistants & Advanced Practice Nursing Providers 363LP0808X,Nurse Practitioner Psych/Mental Health,Physician Assistants & Advanced Practice Nursing Providers 363LP1700X,Nurse Practitioner Perinatal,Physician Assistants & Advanced Practice Nursing Providers 363LP2300X,Nurse Practitioner Primary Care,Physician Assistants & Advanced Practice Nursing Providers @@ -813,13 +813,13 @@ Code,Descriptive_Text,Grouping 364SN0000X,Clinical Nurse Specialist Neonatal,Physician Assistants & Advanced Practice Nursing Providers 364SN0800X,Clinical Nurse Specialist Neuroscience,Physician Assistants & Advanced Practice Nursing Providers 364SP0200X,Clinical Nurse Specialist Pediatrics,Physician Assistants & Advanced Practice Nursing Providers -364SP0807X,Clinical Nurse Specialist Psych/Mental Health Child & Adolescent,Physician Assistants & Advanced Practice Nursing Providers +364SP0807X,"Clinical Nurse Specialist Psych/Mental Health, Child & Adolescent",Physician Assistants & Advanced Practice Nursing Providers 364SP0808X,Clinical Nurse Specialist Psych/Mental Health,Physician Assistants & Advanced Practice Nursing Providers -364SP0809X,Clinical Nurse Specialist Psych/Mental Health Adult,Physician Assistants & Advanced Practice Nursing Providers -364SP0810X,Clinical Nurse Specialist Psych/Mental Health Child & Family,Physician Assistants & Advanced Practice Nursing Providers -364SP0811X,Clinical Nurse Specialist Psych/Mental Health Chronically Ill,Physician Assistants & Advanced Practice Nursing Providers -364SP0812X,Clinical Nurse Specialist Psych/Mental Health Community,Physician Assistants & Advanced Practice Nursing Providers -364SP0813X,Clinical Nurse Specialist Psych/Mental Health Geropsychiatric,Physician Assistants & Advanced Practice Nursing Providers +364SP0809X,"Clinical Nurse Specialist Psych/Mental Health, Adult",Physician Assistants & Advanced Practice Nursing Providers +364SP0810X,"Clinical Nurse Specialist Psych/Mental Health, Child & Family",Physician Assistants & Advanced Practice Nursing Providers +364SP0811X,"Clinical Nurse Specialist Psych/Mental Health, Chronically Ill",Physician Assistants & Advanced Practice Nursing Providers +364SP0812X,"Clinical Nurse Specialist Psych/Mental Health, Community",Physician Assistants & Advanced Practice Nursing Providers +364SP0813X,"Clinical Nurse Specialist Psych/Mental Health, Geropsychiatric",Physician Assistants & Advanced Practice Nursing Providers 364SP1700X,Clinical Nurse Specialist Perinatal,Physician Assistants & Advanced Practice Nursing Providers 364SP2800X,Clinical Nurse Specialist Perioperative,Physician Assistants & Advanced Practice Nursing Providers 364SR0400X,Clinical Nurse Specialist Rehabilitation,Physician Assistants & Advanced Practice Nursing Providers @@ -828,8 +828,8 @@ Code,Descriptive_Text,Grouping 364SW0102X,Clinical Nurse Specialist Women's Health,Physician Assistants & Advanced Practice Nursing Providers 364SX0106X,Clinical Nurse Specialist Occupational Health,Physician Assistants & Advanced Practice Nursing Providers 364SX0200X,Clinical Nurse Specialist Oncology,Physician Assistants & Advanced Practice Nursing Providers -364SX0204X,Clinical Nurse Specialist Oncology Pediatrics,Physician Assistants & Advanced Practice Nursing Providers -367500000X,Nurse Anesthetist Certified Registered ,Physician Assistants & Advanced Practice Nursing Providers +364SX0204X,"Clinical Nurse Specialist Oncology, Pediatrics",Physician Assistants & Advanced Practice Nursing Providers +367500000X,"Nurse Anesthetist, Certified Registered ",Physician Assistants & Advanced Practice Nursing Providers 367A00000X,Advanced Practice Midwife ,Physician Assistants & Advanced Practice Nursing Providers 367H00000X,Anesthesiologist Assistant ,Physician Assistants & Advanced Practice Nursing Providers 372500000X,Chore Provider ,Nursing Service Related Providers @@ -847,10 +847,10 @@ Code,Descriptive_Text,Grouping 376K00000X,Nurse's Aide ,Nursing Service Related Providers 385H00000X,Respite Care ,Respite Care Facility 385HR2050X,Respite Care Respite Care Camp,Respite Care Facility -385HR2055X,Respite Care Respite Care Mental Illness Child,Respite Care Facility -385HR2060X,Respite Care Respite Care Mental Retardation and/or Developmental Disabilities Child,Respite Care Facility -385HR2065X,Respite Care Respite Care Physical Disabilities Child,Respite Care Facility -390200000X,Student in an Organized Health Care Education/Training Program ,Student Health Care +385HR2055X,"Respite Care Respite Care, Mental Illness, Child",Respite Care Facility +385HR2060X,"Respite Care Respite Care, Mental Retardation and/or Developmental Disabilities, Child",Respite Care Facility +385HR2065X,"Respite Care Respite Care, Physical Disabilities, Child",Respite Care Facility +390200000X,Student in an Organized Health Care Education/Training Program ,"Student, Health Care" 405300000X,Prevention Professional ,Other Service Providers NI,No information, UN,Unknown, From b29fb1b9eb301e440e2783c41c46766e3dafedfc Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 11 May 2018 09:44:48 -0500 Subject: [PATCH 394/507] Adding init dependencies --- curated_data/provider_specialty_code.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/curated_data/provider_specialty_code.csv b/curated_data/provider_specialty_code.csv index b502b3f..73f94f4 100644 --- a/curated_data/provider_specialty_code.csv +++ b/curated_data/provider_specialty_code.csv @@ -1,4 +1,4 @@ -Code,Descriptive_Text,Grouping +CODE,DESCRIPTIVE_TEXT,GROUPING 101Y00000X,Counselor,Behavioral Health & Social Service Providers 101YA0400X,Counselor Addiction Substance Use Disorder,Behavioral Health & Social Service Providers 101YM0800X,Counselor Mental Health,Behavioral Health & Social Service Providers From 44f1dc30fa32278cc6d87a4373e7d0ec84366699 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 11 May 2018 10:01:54 -0500 Subject: [PATCH 395/507] Substring on raw provider specialty --- Oracle/patient_chunks_survey.sql | 19 +++++++------------ Oracle/provider.sql | 2 +- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/Oracle/patient_chunks_survey.sql b/Oracle/patient_chunks_survey.sql index 51058d0..ef3cab8 100644 --- a/Oracle/patient_chunks_survey.sql +++ b/Oracle/patient_chunks_survey.sql @@ -1,9 +1,8 @@ -/** patient_chunk_survery - create table to save ntile over patient_num +/** patient_chunks_survery - create table to save ntile over patient_num */ - -whenever sqlerror continue; - drop table patient_chunks; -whenever sqlerror exit; +BEGIN +PMN_DROPSQL('DROP TABLE patient_chunks'); +END; / create table patient_chunks ( chunk_qty integer not null, @@ -15,11 +14,7 @@ create table patient_chunks ( constraint chunk_qty_pos check (chunk_qty > 0), constraint chunk_num_in_range check (chunk_num between 1 and chunk_qty), constraint chunk_first check (patient_num_first is not null or chunk_num = 1) - ); - --- Can we refer to the table without error? ---select coalesce((select 1 from patient_chunks where patient_id_qty > 0 and rownum=1), 1) complete ---from dual; + ) / insert into patient_chunks ( chunk_qty @@ -37,7 +32,7 @@ select :chunk_qty select patient_num, ntile(:chunk_qty) over (order by patient_num) as chunk_num from (select distinct patient_num from BLUEHERONDATA.patient_dimension)) group by chunk_num, :chunk_qty -order by chunk_num; +order by chunk_num / select case when (select count(distinct chunk_num) @@ -46,4 +41,4 @@ select case then 1 else 0 end complete -from dual; +from dual \ No newline at end of file diff --git a/Oracle/provider.sql b/Oracle/provider.sql index 8410be4..64a8748 100644 --- a/Oracle/provider.sql +++ b/Oracle/provider.sql @@ -50,7 +50,7 @@ select cs.prov_id else nvl(sm.specialty, 'UN') end as provider_specialty_primary , cs2.npi , case when cs2.npi is not null then 'Y' else 'N' end as provider_npi_flag - , sp.descriptive_text + , substr(sp.descriptive_text, 1, 50) from clarity.clarity_ser@id cs left join clarity.clarity_ser_2@id cs2 on cs.prov_id = cs2.prov_id left join provider_specialty_map sm on sm.npi = cs2.npi From d89d0ede09a35efe2efe533befa386e3663d97b2 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 11 May 2018 13:09:03 -0500 Subject: [PATCH 396/507] Cod test configuration --- i2p_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index 66201b5..42d6ded 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -284,7 +284,7 @@ class loadSpecialtyCode(LoadCSV): class downloadNPI(CDMStatusTask): taskName = 'NPI_DOWNLOAD' npi_url = 'http://download.cms.gov/nppes/' - dl_path = '/d1/npi/' + dl_path = 'curated_data/' load_path = 'curated_data/' npi_zip = 'NPPES_Data_Dissemination_April_2018.zip' npi_csv = 'npidata_pfile_20050523-20180408.csv' From d361f1c4cc9fdac6de1d5bf468e1038dfb73a4ef Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 11 May 2018 13:20:32 -0500 Subject: [PATCH 397/507] Cod test configuration --- Oracle/pcornet_init.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index df1e414..bc411aa 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -15,7 +15,9 @@ select case when qty = 0 then 1 else 1 end inout_cd_populated from ( ) / -- Make sure the RXNorm mapping table exists -select rxcui from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id where 1=0 +-- TODO : parameterize etl schema in Jenkins +--select rxcui from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id where 1=0 +select 1 from dual / -- Make sure the observation fact medication table is populated select case when qty > 0 then 1 else 1/0 end obs_fact_meds_populated from ( From e6606f126aae32effe51dec00126a68d900db9b3 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 11 May 2018 13:54:39 -0500 Subject: [PATCH 398/507] Cod test configuration --- i2p_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index 42d6ded..8bcf655 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -307,7 +307,7 @@ def fetch(self): with open(self.dl_path + self.npi_zip, 'wb') as fout: fout.write(r.read()) - subprocess.call(['unzip', '-o', self.dl_path + self.npi_zip]) + subprocess.call(['unzip', '-o', self.dl_path + self.npi_zip, '-d', self.dl_path]) def extract(self): self.expectedRecords = 0 From e8fc1fed6b65235dd81ee4ecb3f8f8756221bc02 Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 14 May 2018 16:26:34 -0500 Subject: [PATCH 399/507] Provider relies on provider_dimension rather than clarity@id reads --- Oracle/provider.sql | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Oracle/provider.sql b/Oracle/provider.sql index 64a8748..406050b 100644 --- a/Oracle/provider.sql +++ b/Oracle/provider.sql @@ -42,18 +42,17 @@ PMN_DROPSQL('drop index provider_idx'); execute immediate 'truncate table provider'; insert into provider(providerid, provider_sex, provider_specialty_primary, provider_npi, provider_npi_flag, raw_provider_specialty_primary) -select cs.prov_id - , case when cs.sex = 'U' then 'UN' - when cs.sex is null then 'NI' - else cs.sex end as sex +select pd.provider_id + , case when pd.provider_sex = 'U' then 'UN' + when pd.provider_sex is null then 'NI' + else pd.provider_sex end as sex , case when sm.npi is null then 'NI' else nvl(sm.specialty, 'UN') end as provider_specialty_primary - , cs2.npi - , case when cs2.npi is not null then 'Y' else 'N' end as provider_npi_flag + , pd.provider_npi + , case when pd.provider_npi is not null then 'Y' else 'N' end as provider_npi_flag , substr(sp.descriptive_text, 1, 50) - from clarity.clarity_ser@id cs - left join clarity.clarity_ser_2@id cs2 on cs.prov_id = cs2.prov_id - left join provider_specialty_map sm on sm.npi = cs2.npi + from &&i2b2_data_schema.provider_dimension pd + left join provider_specialty_map sm on sm.npi = pd.provider_npi left join provider_specialty_code sp on sm.specialty = sp.code; execute immediate 'create index provider_idx on provider (PROVIDERID)'; From 2645b6fd3536e6794273141140de44478bf1cd1e Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 18 May 2018 10:11:46 -0500 Subject: [PATCH 400/507] Correcting bad merge --- csv_load.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/csv_load.py b/csv_load.py index 730cec3..e4e13cf 100644 --- a/csv_load.py +++ b/csv_load.py @@ -1,6 +1,7 @@ from collections import defaultdict from csv import DictReader from etl_tasks import CDMStatusTask +from param_val import StrParam from typing import Dict from sqlalchemy import func, MetaData, Table, Column # type: ignore @@ -13,7 +14,6 @@ class LoadCSV(CDMStatusTask): csvname = StrParam() - actual = 0 if actual is None else actual def run(self) -> None: self.setTaskStart() @@ -45,4 +45,3 @@ def sz(l: int, chunk: int=16) -> int: table.create(db) db.execute(table.insert(), l) - db.execute(statusTable.insert(), [{'STATUS': self.tablename, 'LAST_UPDATE': datetime.now(), 'RECORDS': actual}]) From 527e61c82832cc1f376ec24ad9806234a24ab812 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 18 May 2018 10:12:01 -0500 Subject: [PATCH 401/507] Adding type hints --- i2p_tasks.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index 257073a..d3dd16f 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -109,14 +109,14 @@ def requires(self) -> List[luigi.Task]: class obs_clin(CDMScriptTask): script = Script.obs_clin - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] class obs_gen(CDMScriptTask): script = Script.obs_gen - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] @@ -233,7 +233,7 @@ def requires(self) -> List[luigi.Task]: class provider(CDMScriptTask): script = Script.provider - def requires(self): + def requires(self) -> List[luigi.Task]: return [loadSpecialtyMap(), loadSpecialtyCode(), encounter()] @@ -248,7 +248,7 @@ class loadLabNormal(LoadCSV): taskName = 'LABNORMAL' csvname = 'curated_data/labnormal.csv' - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] @@ -256,7 +256,7 @@ class loadHarvestLocal(LoadCSV): taskName = 'HARVEST_LOCAL' csvname = 'curated_data/harvest_local.csv' - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] @@ -265,7 +265,7 @@ class loadLanguage(LoadCSV): # language.csv is a copy of the CDM spec's patient_pref_language_spoke spreadsheet. csvname = 'curated_data/language.csv' - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] @@ -274,7 +274,7 @@ class loadSpecialtyMap(LoadCSV): # provider_specialty_map.csv is created on demand by the downloadNPI method. csvname = 'curated_data/provider_specialty_map.csv' - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init(), downloadNPI()] @@ -298,13 +298,13 @@ class downloadNPI(CDMStatusTask): npi_col = 'NPI' taxonomy_ct = 15 - def run(self): + def run(self) -> None: self.setTaskStart() self.fetch() self.extract() self.setTaskEnd(self.expectedRecords) - def fetch(self): + def fetch(self) -> None: r = urllib.request.urlopen(self.npi_url + self.npi_zip) with open(self.dl_path + self.npi_zip, 'wb') as fout: @@ -312,7 +312,7 @@ def fetch(self): subprocess.call(['unzip', '-o', self.dl_path + self.npi_zip, '-d', self.dl_path]) - def extract(self): + def extract(self) -> None: self.expectedRecords = 0 with open(self.dl_path + self.npi_csv, 'r') as fin: with open(self.load_path + self.specialty_csv, 'w', newline='') as fout: From b4c990a2ad6c73f682391a394f3b06c0668fb804 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 30 May 2018 15:52:47 -0500 Subject: [PATCH 402/507] Optimize Luigi ops, optimize prescribing table creation, basic provider table creation --- Oracle/condition.sql | 2 +- Oracle/death.sql | 4 +- Oracle/death_cause.sql | 3 +- Oracle/demographic.sql | 2 +- Oracle/diagnosis.sql | 3 +- Oracle/dispensing.sql | 3 +- Oracle/encounter.sql | 3 +- Oracle/enrollment.sql | 3 +- Oracle/lab_result_cm.sql | 2 +- Oracle/med_admin.sql | 2 +- Oracle/obs_clin.sql | 2 +- Oracle/obs_gen.sql | 2 +- Oracle/pcornet_init.sql | 9 +-- Oracle/pcornet_loader.sql | 2 +- Oracle/prescribing.sql | 21 ++++- Oracle/pro_cm.sql | 2 +- Oracle/procedures.sql | 3 +- Oracle/provider.sql | 2 +- Oracle/vital.sql | 3 +- client.cfg | 67 ++++++---------- csv_load.py | 10 ++- etl_tasks.py | 42 ++++++++-- i2p_tasks.py | 164 +++++++++++++++++++++++++++----------- logging.cfg | 8 +- 24 files changed, 229 insertions(+), 135 deletions(-) diff --git a/Oracle/condition.sql b/Oracle/condition.sql index 3bf9114..b572662 100644 --- a/Oracle/condition.sql +++ b/Oracle/condition.sql @@ -108,4 +108,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from condition) where task = 'condition' / -select 1 from cdm_status where task = 'condition' \ No newline at end of file +select records from cdm_status where task = 'condition' \ No newline at end of file diff --git a/Oracle/death.sql b/Oracle/death.sql index 595d732..9c2aaeb 100644 --- a/Oracle/death.sql +++ b/Oracle/death.sql @@ -51,6 +51,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from death) where task = 'death' / -select 1 from cdm_status where task = 'death' - ---SELECT count(PATID) from death where rownum = 1 \ No newline at end of file +select records from cdm_status where task = 'death' \ No newline at end of file diff --git a/Oracle/death_cause.sql b/Oracle/death_cause.sql index 4f61aeb..c3f5df7 100644 --- a/Oracle/death_cause.sql +++ b/Oracle/death_cause.sql @@ -19,5 +19,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from death_cause) where task = 'death_cause' / -select 1 from cdm_status where task = 'death_cause' ---SELECT count(PATID) from death_cause where rownum = 1 \ No newline at end of file +select records + 1 from cdm_status where task = 'death_cause' \ No newline at end of file diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql index 972bd61..44514cb 100644 --- a/Oracle/demographic.sql +++ b/Oracle/demographic.sql @@ -214,4 +214,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from demographic) where task = 'demographic' / -select 1 from cdm_status where task = 'demographic' \ No newline at end of file +select records from cdm_status where task = 'demographic' \ No newline at end of file diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 342dc27..d303d0c 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -251,5 +251,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from diagnosis) where task = 'diagnosis' / -select 1 from cdm_status where task = 'diagnosis' ---SELECT count(DIAGNOSISID) from diagnosis where rownum = 1 \ No newline at end of file +select records from cdm_status where task = 'diagnosis' \ No newline at end of file diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index 992cd61..a538110 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -184,5 +184,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from dispensing) where task = 'dispensing' / -select 1 from cdm_status where task = 'dispensing' ---SELECT count(DISPENSINGID) from dispensing where rownum = 1 \ No newline at end of file +select records from cdm_status where task = 'dispensing' \ No newline at end of file diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 3f9ea72..f581993 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -105,5 +105,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from encounter) where task = 'encounter' / -select 1 from cdm_status where task = 'encounter' ---SELECT count(ENCOUNTERID) from encounter where rownum = 1 \ No newline at end of file +select records from cdm_status where task = 'encounter' \ No newline at end of file diff --git a/Oracle/enrollment.sql b/Oracle/enrollment.sql index 07d1ddb..9569479 100644 --- a/Oracle/enrollment.sql +++ b/Oracle/enrollment.sql @@ -56,5 +56,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from enrollment) where task = 'enrollment' / -select 1 from cdm_status where task = 'enrollment' ---SELECT count(PATID) from enrollment where rownum = 1 \ No newline at end of file +select records from cdm_status where task = 'enrollment' \ No newline at end of file diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 48e56f4..474d7c7 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -171,4 +171,4 @@ set end_time = sysdate, records = (select count(*) from lab_result_cm) where task = 'lab_result_cm' / -select 1 from cdm_status where task = 'lab_result_cm' \ No newline at end of file +select records from cdm_status where task = 'lab_result_cm' \ No newline at end of file diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 39623c6..293b38d 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -118,4 +118,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from med_admin) where task = 'med_admin' / -select 1 from cdm_status where task = 'med_admin' +select records from cdm_status where task = 'med_admin' diff --git a/Oracle/obs_clin.sql b/Oracle/obs_clin.sql index 59c01eb..d030694 100644 --- a/Oracle/obs_clin.sql +++ b/Oracle/obs_clin.sql @@ -33,4 +33,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from obs_clin) where task = 'obs_clin' / -select 1 from cdm_status where task = 'obs_clin' \ No newline at end of file +select records + 1 from cdm_status where task = 'obs_clin' \ No newline at end of file diff --git a/Oracle/obs_gen.sql b/Oracle/obs_gen.sql index 0c20a33..37b8fb4 100644 --- a/Oracle/obs_gen.sql +++ b/Oracle/obs_gen.sql @@ -33,4 +33,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from obs_gen) where task = 'obs_gen' / -select 1 from cdm_status where task = 'obs_gen' \ No newline at end of file +select records + 1 from cdm_status where task = 'obs_gen' \ No newline at end of file diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index bc411aa..0f8af64 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -9,15 +9,12 @@ select providerid from "&&i2b2_data_schema".visit_dimension where 1=0 / -- Make sure the inout_cd has been populated -- See heron_encounter_style.sql --- TODO: restore 1/0 check. -select case when qty = 0 then 1 else 1 end inout_cd_populated from ( +select case when qty = 0 then 1/0 else 1 end inout_cd_populated from ( select count(*) qty from "&&i2b2_data_schema".visit_dimension where inout_cd is not null ) / -- Make sure the RXNorm mapping table exists --- TODO : parameterize etl schema in Jenkins ---select rxcui from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id where 1=0 -select 1 from dual +select rxcui from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id where 1=0 / -- Make sure the observation fact medication table is populated select case when qty > 0 then 1 else 1/0 end obs_fact_meds_populated from ( @@ -252,4 +249,4 @@ set end_time = sysdate, records = 0 where task = 'pcornet_init' / -select 1 from cdm_status where task = 'pcornet_init' +select records + 1 from cdm_status where task = 'pcornet_init' diff --git a/Oracle/pcornet_loader.sql b/Oracle/pcornet_loader.sql index a260b42..2fb7b8f 100644 --- a/Oracle/pcornet_loader.sql +++ b/Oracle/pcornet_loader.sql @@ -60,4 +60,4 @@ update cdm_status set end_time = sysdate, records = 0 where task = 'pcornet_loader' / -select 1 from cdm_status where task = 'pcornet_loader' \ No newline at end of file +select records + 1 from cdm_status where task = 'pcornet_loader' \ No newline at end of file diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 43204db..e8c4db8 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -191,6 +191,21 @@ left join ) basis on basis.instance_num = rx.instance_num and basis.concept_cd = rx.concept_cd / +create table prescribing_w_dose as +select rx.* +, units_cd rx_dose_ordered +, nval_num rx_dose_ordered_unit +from prescribing_w_basis rx +left join + (select instance_num + , concept_cd + , case when units_cd = 'mcg' then 'ug' when units_cd = 'l' then 'ml' else units_cd end units_cd + , case when units_cd = 'l' then nval_num * 1000 else nval_num end nval_num + from blueherondata.observation_fact + where modifier_cd in ('MedObs:Dose|mg', 'MedObs:Dose|meq', 'MedObs:Dose|l') + ) dose on dose.instance_num = rx.instance_num and dose.concept_cd = rx.concept_cd +/ + create table prescribing as select rx.prescribingid , rx.patient_num patid @@ -200,6 +215,8 @@ select rx.prescribingid , to_char(rx.start_date, 'HH24:MI') rx_order_time , trunc(rx.start_date) rx_start_date , trunc(rx.end_date) rx_end_date +, rx.rx_dose_ordered +, rx.rx_dose_ordered_unit , rx.rx_quantity , 'NI' rx_quantity_unit , rx.rx_refills @@ -214,7 +231,7 @@ select rx.prescribingid , rx.raw_rxnorm_cui /* ISSUE: HERON should have an actual order time. idea: store real difference between order date start data, possibly using the update date */ -from prescribing_w_basis rx +from prescribing_w_dose rx / create index prescribing_idx on prescribing (PATID, ENCOUNTERID) @@ -229,4 +246,4 @@ set end_time = sysdate, records = (select count(*) from prescribing) where task = 'prescribing' / -select 1 from cdm_status where task = 'prescribing' +select records from cdm_status where task = 'prescribing' diff --git a/Oracle/pro_cm.sql b/Oracle/pro_cm.sql index bb3b5dd..e15701a 100644 --- a/Oracle/pro_cm.sql +++ b/Oracle/pro_cm.sql @@ -41,4 +41,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from pro_cm) where task = 'pro_cm' / -select 1 from cdm_status where task = 'pro_cm' +select records + 1 from cdm_status where task = 'pro_cm' diff --git a/Oracle/procedures.sql b/Oracle/procedures.sql index 095e2d4..f121c3a 100644 --- a/Oracle/procedures.sql +++ b/Oracle/procedures.sql @@ -66,5 +66,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from procedures) where task = 'procedures' / -select 1 from cdm_status where task = 'procedures' ---SELECT count(PATID) from procedures where rownum = 1 \ No newline at end of file +select records from cdm_status where task = 'procedures' \ No newline at end of file diff --git a/Oracle/provider.sql b/Oracle/provider.sql index 406050b..783bd37 100644 --- a/Oracle/provider.sql +++ b/Oracle/provider.sql @@ -71,4 +71,4 @@ set end_time = sysdate, records = (select count(*) from provider) where task = 'provider' / -select 1 from cdm_status where task = 'provider' +select records from cdm_status where task = 'provider' diff --git a/Oracle/vital.sql b/Oracle/vital.sql index d498f75..06271cc 100644 --- a/Oracle/vital.sql +++ b/Oracle/vital.sql @@ -141,5 +141,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from vital) where task = 'vital' / -select 1 from cdm_status where task = 'vital' ---SELECT count(VITALID) from vital where rownum = 1 +select records from cdm_status where task = 'vital' diff --git a/client.cfg b/client.cfg index 2283c0e..580224b 100644 --- a/client.cfg +++ b/client.cfg @@ -16,38 +16,9 @@ logging_conf_file = logging.cfg # oracle://username:password@localhost:5555/database_sid # # RIGHT: -account=oracle://grouse_etl_1@dbhost2/database_sid -passkey=GROUSE_ETL_1_ON_DBHOST2 -ssh_tunnel=localhost:4768 - - -[CLARITYExtract] -# When was it downloaded? -# e.g. suppose a jenkins download job was http://.../job/cms_syn_dl/8 -# and it finished Jul 30, 2015 8:54:25 AM. The we have: -download_date=1487378515445 - -# In order to get build dates from jenkins to luigi, i.e. -# from groovy to python, we use integers, since date interchange -# is a pain. -# -# Using the Jenkins API http://javadoc.jenkins.io/ -# A bit of groovy like this gets what we need: -# dlBuild?.timestamp.getTimeInMillis() + dlBuild?.duration - -# TODO: Split work into how many chunks by bene_id? -# bene_chunks = 64 - - -[NightHeronCreate] -# Where did we (or should we) create an i2b2 project? -# ref i2b2 sources: -# crc_create_datamart_oracle.sql -# crc_create_uploader_oracle.sql -star_schema = NIGHTHERONDATA - -# And what i2b2 project_id? -project_id = BlueHeron +account=oracle://pcornet_cdm@localhost:8521/STAGEDEV +passkey=BMID_KEY +ssh_tunnel= #[scheduler] # ISSUE: task history? @@ -57,13 +28,25 @@ project_id = BlueHeron encounter_mapping=1 patient_mapping=1 -[retcode] -# see also http://luigi.readthedocs.io/en/stable/configuration.html -# The following return codes are the recommended exit codes for Luigi -# They are in increasing level of severity (for most applications) -already_running=10 -missing_data=20 -not_run=25 -task_failed=30 -scheduling_error=35 -unhandled_exception=40 \ No newline at end of file +[I2PConfig] +datamart_id = C4UK +datamart_name = University of Kansas +enrollment_months_back = 42 +i2b2_data_schema = BLUEHERONDATA +i2b2_etl_schema = HERON_ETL_3 +i2b2_meta_schema = BLUEHERONMETADATA +min_pat_list_date_dd_mon_rrrr = 01-Jan-2010 +min_visit_date_dd_mon_rrrr = 01-Jan-2010 +network_id = C4 +network_name = GPC + +[NPIDownloadConfig] +dl_path = curated_data/ +extract_path = curated_data/ +npi_csv = npidata_pfile_20050523-20180408.csv +npi_url = http://download.cms.gov/nppes/ +npi_zip = NPPES_Data_Dissemination_April_2018.zip +taxonomy_col = Healthcare Provider Taxonomy Code_ +switch_col = Healthcare Provider Primary Taxonomy Switch_ +npi_col = NPI +taxonomy_ct = 15 \ No newline at end of file diff --git a/csv_load.py b/csv_load.py index e4e13cf..1b6e245 100644 --- a/csv_load.py +++ b/csv_load.py @@ -13,12 +13,18 @@ log = logging.getLogger(__name__) class LoadCSV(CDMStatusTask): + ''' + Creates a table in the db with the name taskName. + + The table is loaded with data from the csv file specifed by csvname. + Table creation is logged in the cdm status table. + ''' csvname = StrParam() def run(self) -> None: self.setTaskStart() self.load() - self.setTaskEnd(self.getRecordCount()) + self.setTaskEnd(self.getRecordCountFromTable()) def load(self) -> None: def sz(l: int, chunk: int=16) -> int: @@ -33,6 +39,8 @@ def sz(l: int, chunk: int=16) -> int: Dict # for tools that don't see type: comments. mcl = defaultdict(int) # type: Dict[str, int] + # Iterate data in the input csv, find the largest data item + # and set the column size to the item size rounded to the nearest chunk. for row in dr: l.append(row) for col in dr.fieldnames: diff --git a/etl_tasks.py b/etl_tasks.py index 5b4003a..c610978 100644 --- a/etl_tasks.py +++ b/etl_tasks.py @@ -365,27 +365,57 @@ def bulk_insert(self, conn: LoggedConnection, fname: str, line: int, class CDMStatusTask(DBAccessTask): + ''' + A DBAccessTask that relies on the CDM status table to assess completion. + + Typical usage is to record the start of the task, run task operations and then record + the end of the task. + self.setTaskStart() + self.load() + self.setTaskEnd(self.getRecordCountFromTable()) + ''' taskName = StrParam() + + # Basic status check, assume the typical task produces at least one record. expectedRecords = IntParam(default=1) def complete(self) -> bool: + ''' + Complete when the CDM status table reports at least as many records as expected for the task. + ''' with self.connection() as q: - actualRecords = q.scalar('select records from cdm_status where task = \'%s\'' % self.taskName) - actualRecords = 0 if actualRecords == None else actualRecords - log.info('task %s has %d rows', self.taskName, actualRecords) - return actualRecords >= self.expectedRecords # type: ignore # sqla + statusTableRecordCount = q.scalar('select records from cdm_status where task = \'%s\'' % self.taskName) + + # If true, the task has not been logged in the CDM status table or has been logged and is in an + # inconsistent state with the number of records set to null. + if statusTableRecordCount == None: + return False - def getRecordCount(self) -> int: + log.info('task %s has %d rows', self.taskName, statusTableRecordCount) + return statusTableRecordCount >= self.expectedRecords # type: ignore # sqla + + def getRecordCountFromTable(self) -> int: + ''' + Queries the database for the number of records in the table named taskName + ''' + # This op is out of sync with the rest of the class, in that it assumes + # the task must represent the creation of a table in the db. with self.connection() as q: return q.scalar(sqla.select([func.count()]).select_from(self.taskName)) - def setTaskEnd(self, rowCount: int): + def setTaskEnd(self, rowCount: int) -> None: + ''' + Updates the taskName entry in the CDM status table with an end time of now and a count of records. + ''' statusTable = Table("cdm_status", MetaData(), Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) db = self._dbtarget().engine db.execute(statusTable.update().where(statusTable.c.TASK == self.taskName), [{'END_TIME': datetime.now(), 'RECORDS': rowCount}]) def setTaskStart(self) -> None: + ''' + Adds taskName to the CDM status table with a start time of now. + ''' statusTable = Table("cdm_status", MetaData(), Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) db = self._dbtarget().engine diff --git a/i2p_tasks.py b/i2p_tasks.py index d3dd16f..6b60079 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -8,82 +8,98 @@ from csv_load import LoadCSV from etl_tasks import CDMStatusTask, SqlScriptTask -from param_val import IntParam +from param_val import IntParam, StrParam from script_lib import Script from sql_syntax import Environment, Params import csv import subprocess import urllib.request -import zipfile -class CDMScriptTask(SqlScriptTask): + +class I2PConfig(luigi.Config): + datamart_id = StrParam(description='see client.cfg') + datamart_name = StrParam(description='see client.cfg') + enrollment_months_back = StrParam(description='see client.cfg') + i2b2_data_schema = StrParam(description='see client.cfg') + i2b2_etl_schema = StrParam(description='see client.cfg') + i2b2_meta_schema = StrParam(description='see client.cfg') + min_pat_list_date_dd_mon_rrrr = StrParam(description='see client.cfg') + min_visit_date_dd_mon_rrrr = StrParam(description='see client.cfg') + network_id = StrParam(description='see client.cfg') + network_name = StrParam(description='see client.cfg') + + +class I2PScriptTask(SqlScriptTask): @property def variables(self) -> Environment: - return dict(datamart_id='C4UK', datamart_name='University of Kansas', i2b2_data_schema='BLUEHERONDATA', - min_pat_list_date_dd_mon_rrrr='01-Jan-2010', min_visit_date_dd_mon_rrrr='01-Jan-2010', - i2b2_meta_schema='BLUEHERONMETADATA', enrollment_months_back='42', network_id='C4', - network_name='GPC', i2b2_etl_schema='HERON_ETL_3') + return dict(datamart_id=I2PConfig().datamart_id, datamart_name=I2PConfig().datamart_name, + i2b2_data_schema=I2PConfig().i2b2_data_schema, + min_pat_list_date_dd_mon_rrrr=I2PConfig().min_pat_list_date_dd_mon_rrrr, + min_visit_date_dd_mon_rrrr=I2PConfig().min_visit_date_dd_mon_rrrr, + i2b2_meta_schema=I2PConfig().i2b2_meta_schema, + enrollment_months_back=I2PConfig().enrollment_months_back, network_id=I2PConfig().network_id, + network_name=I2PConfig().network_name, i2b2_etl_schema=I2PConfig().i2b2_etl_schema) -class condition(CDMScriptTask): +class condition(I2PScriptTask): script = Script.condition def requires(self) -> List[luigi.Task]: return [encounter()] -class death(CDMScriptTask): +class death(I2PScriptTask): script = Script.death def requires(self) -> List[luigi.Task]: return [demographic()] -class death_cause(CDMScriptTask): +class death_cause(I2PScriptTask): script = Script.death_cause def requires(self) -> List[luigi.Task]: return [pcornet_init()] -class demographic(CDMScriptTask): +class demographic(I2PScriptTask): script = Script.demographic def requires(self) -> List[luigi.Task]: return [loadLanguage(), pcornet_init()] -class diagnosis(CDMScriptTask): +class diagnosis(I2PScriptTask): script = Script.diagnosis def requires(self) -> List[luigi.Task]: return [encounter()] -class dispensing(CDMScriptTask): +class dispensing(I2PScriptTask): script = Script.dispensing def requires(self) -> List[luigi.Task]: return [encounter()] -class encounter(CDMScriptTask): +class encounter(I2PScriptTask): script = Script.encounter def requires(self) -> List[luigi.Task]: return [demographic()] -class enrollment(CDMScriptTask): +class enrollment(I2PScriptTask): script = Script.enrollment def requires(self) -> List[luigi.Task]: return [pcornet_init()] -class harvest(CDMScriptTask): +class harvest(I2PScriptTask): script = Script.harvest def requires(self) -> List[luigi.Task]: @@ -92,35 +108,35 @@ def requires(self) -> List[luigi.Task]: prescribing(), pro_cm(), procedures(), provider(), vital()] -class lab_result_cm(CDMScriptTask): +class lab_result_cm(I2PScriptTask): script = Script.lab_result_cm def requires(self) -> List[luigi.Task]: return [encounter(), loadLabNormal()] -class med_admin(CDMScriptTask): +class med_admin(I2PScriptTask): script = Script.med_admin def requires(self) -> List[luigi.Task]: return [pcornet_init()] -class obs_clin(CDMScriptTask): +class obs_clin(I2PScriptTask): script = Script.obs_clin def requires(self) -> List[luigi.Task]: return [pcornet_init()] -class obs_gen(CDMScriptTask): +class obs_gen(I2PScriptTask): script = Script.obs_gen def requires(self) -> List[luigi.Task]: return [pcornet_init()] -class CDMPatientGroupTask(CDMScriptTask): +class I2PPatientGroupTask(I2PScriptTask): patient_num_first = IntParam() patient_num_last = IntParam() patient_num_qty = IntParam(significant=False, default=-1) @@ -133,7 +149,7 @@ def run(self) -> None: class _PatientNumGrouped(luigi.WrapperTask): - group_tasks = cast(List[Type[CDMPatientGroupTask]], []) # abstract + group_tasks = cast(List[Type[I2PPatientGroupTask]], []) # abstract def requires(self) -> List[luigi.Task]: deps = [] # type: List[luigi.Task] @@ -188,56 +204,56 @@ def results(self) -> List[RowProxy]: return [] -class pcornet_init(CDMScriptTask): +class pcornet_init(I2PScriptTask): script = Script.pcornet_init def requires(self) -> List[luigi.Task]: return [] -class pcornet_loader(CDMScriptTask): +class pcornet_loader(I2PScriptTask): script = Script.pcornet_loader def requires(self) -> List[luigi.Task]: return [harvest()] -class pcornet_trial(CDMScriptTask): +class pcornet_trial(I2PScriptTask): script = Script.pcornet_trial def requires(self) -> List[luigi.Task]: return [pcornet_init()] -class prescribing(CDMScriptTask): +class prescribing(I2PScriptTask): script = Script.prescribing def requires(self) -> List[luigi.Task]: return [encounter()] -class pro_cm(CDMScriptTask): +class pro_cm(I2PScriptTask): script = Script.pro_cm def requires(self) -> List[luigi.Task]: return [pcornet_init()] -class procedures(CDMScriptTask): +class procedures(I2PScriptTask): script = Script.procedures def requires(self) -> List[luigi.Task]: return [encounter()] -class provider(CDMScriptTask): +class provider(I2PScriptTask): script = Script.provider def requires(self) -> List[luigi.Task]: return [loadSpecialtyMap(), loadSpecialtyCode(), encounter()] -class vital(CDMScriptTask): +class vital(I2PScriptTask): script = Script.vital def requires(self) -> List[luigi.Task]: @@ -262,46 +278,65 @@ def requires(self) -> List[luigi.Task]: class loadLanguage(LoadCSV): taskName = 'LANGUAGE_CODE' - # language.csv is a copy of the CDM spec's patient_pref_language_spoke spreadsheet. + # language.csv is a copy of the CDM spec's patient_pref_language_spoken spreadsheet. + # It maps a language code to descriptive text. When the spreadsheet mapped more than + # one text value to a code, duplicate codes where created in language.csv. csvname = 'curated_data/language.csv' def requires(self) -> List[luigi.Task]: return [pcornet_init()] +class NPIDownloadConfig(luigi.Config): + # The configured 'path' and 'npi' variables are used by the downloadNPI method to fetch and + # store the NPPES zip file. Changes to these may require changes to the file system. + dl_path = StrParam(description='Path where the NPPES zip file will be stored and unzipped.') + extract_path = StrParam(description='Path where the extract') + npi_csv = StrParam(description='CSV file in the NPPES zip that contains NPI data.') + npi_url = StrParam(description='URL for the NPPES download site.') + npi_zip = StrParam(description='Name of the NPPES zip file.') + + # The configured 'col' and 'ct' variables reflect the layout of the NPI data file. + # The extracNPI method uses these values to parse the NPI data file. + # Changes to these may require code changes. + taxonomy_col = StrParam(description='Header for the taxonomy columns in the NPI data file.') + switch_col = StrParam(description='Header for the switch columns in the NPI data file.') + npi_col = StrParam(description='Header for the NPI column in the NPI data file.') + taxonomy_ct = IntParam(description='Number of taxonomy columns in the NPI data file.') + + class loadSpecialtyMap(LoadCSV): taskName = 'PROVIDER_SPECIALTY_MAP' - # provider_specialty_map.csv is created on demand by the downloadNPI method. + # provider_specialty_map.csv is created on demand by the extractNPI method. + # It maps a National Provider Identifier (NPI) to a primary provider specialty code. csvname = 'curated_data/provider_specialty_map.csv' def requires(self) -> List[luigi.Task]: - return [pcornet_init(), downloadNPI()] + return [pcornet_init(), extractNPI()] class loadSpecialtyCode(LoadCSV): taskName = 'PROVIDER_SPECIALTY_CODE' # provider_specialty_code.csv is a copy of the CDM spec's provider_primary_specialty spreadsheet. + # It maps a provider specialty code to a descriptive text and grouping. csvname = 'curated_data/provider_specialty_code.csv' class downloadNPI(CDMStatusTask): + ''' + Download the NPPES zip file and extract the NPI data file. + ''' taskName = 'NPI_DOWNLOAD' - npi_url = 'http://download.cms.gov/nppes/' - dl_path = 'curated_data/' - load_path = 'curated_data/' - npi_zip = 'NPPES_Data_Dissemination_April_2018.zip' - npi_csv = 'npidata_pfile_20050523-20180408.csv' - specialty_csv = 'provider_specialty_map.csv' + expectedRecords = 0 - taxonomy_col = 'Healthcare Provider Taxonomy Code_' - switch_col = 'Healthcare Provider Primary Taxonomy Switch_' - npi_col = 'NPI' - taxonomy_ct = 15 + dl_path = NPIDownloadConfig().dl_path + npi_url = NPIDownloadConfig().npi_url + npi_zip = NPIDownloadConfig().npi_zip def run(self) -> None: self.setTaskStart() self.fetch() - self.extract() + self.unzip() self.setTaskEnd(self.expectedRecords) def fetch(self) -> None: @@ -310,24 +345,57 @@ def fetch(self) -> None: with open(self.dl_path + self.npi_zip, 'wb') as fout: fout.write(r.read()) - subprocess.call(['unzip', '-o', self.dl_path + self.npi_zip, '-d', self.dl_path]) + def unzip(self) -> None: + subprocess.call(['unzip', '-o', self.dl_path + self.npi_zip, '-d', self.dl_path]) # ISSUE: ambient + + +class extractNPI(CDMStatusTask): + ''' + Extract the Nation Provider Identifier (NPI) and primary specialty from the NPPES download file + and write the data to a separate csv file. + ''' + taskName = 'NPI_EXTRACT' + specialty_csv = 'provider_specialty_map.csv' + + dl_path = NPIDownloadConfig().dl_path + extract_path = NPIDownloadConfig().extract_path + npi_col = NPIDownloadConfig().npi_col + npi_csv = NPIDownloadConfig().npi_csv + switch_col = NPIDownloadConfig().switch_col + taxonomy_col = NPIDownloadConfig().taxonomy_col + + taxonomy_ct = NPIDownloadConfig().taxonomy_ct + + def run(self) -> None: + self.setTaskStart() + self.extract() + self.setTaskEnd(self.expectedRecords) + + def requires(self) -> List[luigi.Task]: + return [downloadNPI()] def extract(self) -> None: + self.expectedRecords = 0 - with open(self.dl_path + self.npi_csv, 'r') as fin: - with open(self.load_path + self.specialty_csv, 'w', newline='') as fout: + + with open(self.dl_path + self.npi_csv, 'r', encoding='utf-8') as fin: + with open(self.extract_path + self.specialty_csv, 'w', newline='') as fout: reader = csv.DictReader(fin) writer = csv.writer(fout) writer.writerow(['NPI', 'SPECIALTY']) for row in reader: self.expectedRecords = self.expectedRecords + 1 useDefault = True + # Search the taxonomy columns for a primary specialty. for i in range(1, self.taxonomy_ct + 1): + # 'Y' in the switch column indicates that the current taxonomy column contains + # the primary specialty. if row[self.switch_col + str(i)] == 'Y': useDefault = False writer.writerow([row[self.npi_col], row[self.taxonomy_col + str(i)]]) continue + # If true, the NPI file did not explicitly identify a primary specialty. Use the value + # in taxonomy column one as the default. if (useDefault): writer.writerow([row[self.npi_col], row[self.taxonomy_col + str(1)]]) - diff --git a/logging.cfg b/logging.cfg index 798c566..6d305bb 100644 --- a/logging.cfg +++ b/logging.cfg @@ -13,8 +13,8 @@ keys=root,luigi_debug [logger_root] level=INFO -#handlers=console,detail -handlers=console +handlers=console,detail +#handlers=console [logger_luigi_debug] level=DEBUG @@ -22,8 +22,8 @@ handlers=luigi_debug qualname=luigi-interface [handlers] -#keys=console,detail,luigi_debug -keys=console,luigi_debug +keys=console,detail,luigi_debug +#keys=console,luigi_debug [handler_console] level=INFO From e57b26a2899e2be670b04866fcee90aeb885959e Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 30 May 2018 16:29:25 -0500 Subject: [PATCH 403/507] Cleanup prescribing_w_dose table --- Oracle/prescribing.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index e8c4db8..135b49a 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -35,6 +35,10 @@ PMN_DROPSQL('drop table prescribing_w_basis'); END; / BEGIN +PMN_DROPSQL('drop table prescribing_w_dose'); +END; +/ +BEGIN PMN_DROPSQL('DROP sequence prescribing_seq'); END; / From 097abc0bdc0508c122d3f0336fc9134ebb2aa308 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 20 Jun 2018 14:56:20 -0500 Subject: [PATCH 404/507] Map specimen source to pcornet values and add to lab_results_cm --- Oracle/demographic.sql | 3 ++ Oracle/lab_result_cm.sql | 80 ++++++++++++++++++++++++---------------- i2p_tasks.py | 8 ++++ 3 files changed, 60 insertions(+), 31 deletions(-) diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql index 44514cb..b1ac6d9 100644 --- a/Oracle/demographic.sql +++ b/Oracle/demographic.sql @@ -190,6 +190,9 @@ FETCH getsql INTO sqltext; END LOOP; CLOSE getsql; +-- Logic of the using statment is - +-- When the source (i2b2patient) doesn't provide a lanaguage_cd, substitute 'no information', which will be coded +-- as 'NI'. When the source provides a language_cd that isn't in the mapping table, substitute the code 'OT'. merge into demographic d using ( select NVL(code, 'OT') as code, language_cd, patient_num diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 474d7c7..7143581 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -27,6 +27,10 @@ PMN_DROPSQL('drop table lab_result_w_norm'); END; / +BEGIN +PMN_DROPSQL('drop table lab_result_w_source'); +END; + BEGIN PMN_DROPSQL('DROP SEQUENCE lab_result_cm_seq'); END; @@ -52,6 +56,7 @@ create table lab_result_key as select lab_result_cm_seq.nextval LAB_RESULT_CM_ID , patient_num PATID , encounter_num ENCOUNTERID +, instance_num INSTANCE_NUM , provider_id PROVIDERID , start_date LAB_ORDER_DATE , end_date RESULT_DATE @@ -71,13 +76,12 @@ and (m.nval_num is null or m.nval_num<=9999999) create table lab_result_w_base as select lab.* -, pl.pcori_specimen_source SPECIMEN_SOURCE , pl.pcori_basecode LAB_LOINC , pl.c_path C_PATH from lab_result_key lab -inner join +left join ( - select c_basecode, c_path, pcori_basecode, pcori_specimen_source + select c_basecode, c_path, pcori_basecode from pcornet_lab where c_fullname like '\PCORI\LAB_RESULT_CM\%' ) pl @@ -88,7 +92,7 @@ create table lab_result_w_parent as select lab.* , parent.c_basecode LAB_NAME from lab_result_w_base lab -inner join +left join ( select c_fullname, c_basecode from pcornet_lab @@ -114,48 +118,62 @@ and (lab.LAB_ORDER_DATE - norm.birth_date) > norm.age_lower and (lab.LAB_ORDER_DATE - norm.birth_date) <= norm.age_upper / +create table lab_result_w_source as +select lab.* +, NVL(code, 'NI') SPECIMEN_SOURCE +from lab_result_w_norm lab +left join +( + select distinct instance_num, NVL(code, 'OT') code + from &&i2b2_data_schema.supplemental_fact sf + left join specimen_source_map ssm on lower(sf.tval_char) = lower(ssm.specimen_source_name) + and ssm.specimen_source_name is not null +) map +on lab.instance_num = map.instance_num +/ + create table lab_result_cm as -select distinct cast(norm.LAB_RESULT_CM_ID as varchar(19)) LAB_RESULT_CM_ID -, cast(norm.PATID as varchar(50)) PATID -, cast(norm.ENCOUNTERID as varchar(50)) ENCOUNTERID -, case when norm.LAB_NAME like 'LAB_NAME%' then substr(norm.LAB_NAME, 10, 10) else 'UN' end LAB_NAME -, case when norm.SPECIMEN_SOURCE like '%or SR_PLS' then 'SR_PLS' when norm.SPECIMEN_SOURCE is null then 'NI' else norm.SPECIMEN_SOURCE end SPECIMEN_SOURCE -, nvl(norm.LAB_LOINC, 'NI') LAB_LOINC +select distinct cast(lab.LAB_RESULT_CM_ID as varchar(19)) LAB_RESULT_CM_ID +, cast(lab.PATID as varchar(50)) PATID +, cast(lab.ENCOUNTERID as varchar(50)) ENCOUNTERID +, case when lab.LAB_NAME like 'LAB_NAME%' then substr(lab.LAB_NAME, 10, 10) else 'UN' end LAB_NAME +, lab.SPECIMEN_SOURCE +, nvl(lab.LAB_LOINC, 'NI') LAB_LOINC , 'NI' PRIORITY , 'NI' RESULT_LOC -, nvl(norm.LAB_LOINC, 'NI') LAB_PX +, nvl(lab.LAB_LOINC, 'NI') LAB_PX , 'LC' LAB_PX_TYPE -, norm.LAB_ORDER_DATE LAB_ORDER_DATE -, norm.LAB_ORDER_DATE SPECIMEN_DATE -, to_char(norm.LAB_ORDER_DATE, 'HH24:MI') SPECIMEN_TIME -, norm.RESULT_DATE -, to_char(norm.RESULT_DATE, 'HH24:MI') RESULT_TIME +, lab.LAB_ORDER_DATE LAB_ORDER_DATE +, lab.LAB_ORDER_DATE SPECIMEN_DATE +, to_char(lab.LAB_ORDER_DATE, 'HH24:MI') SPECIMEN_TIME +, lab.RESULT_DATE +, to_char(lab.RESULT_DATE, 'HH24:MI') RESULT_TIME , 'NI' RESULT_QUAL -, case when norm.RAW_RESULT = 'N' then norm.RESULT_NUM else null end RESULT_NUM -, case when norm.RAW_RESULT = 'N' then (case nvl(nullif(norm.RESULT_MODIFIER, ''),'NI') when 'E' then 'EQ' when 'NE' then 'OT' when 'L' then 'LT' when 'LE' then 'LE' when 'G' then 'GT' when 'GE' then 'GE' else 'NI' end) else 'TX' end RESULT_MODIFIER +, case when lab.RAW_RESULT = 'N' then lab.RESULT_NUM else null end RESULT_NUM +, case when lab.RAW_RESULT = 'N' then (case nvl(nullif(lab.RESULT_MODIFIER, ''),'NI') when 'E' then 'EQ' when 'NE' then 'OT' when 'L' then 'LT' when 'LE' then 'LE' when 'G' then 'GT' when 'GE' then 'GE' else 'NI' end) else 'TX' end RESULT_MODIFIER , case - when instr(norm.RESULT_UNIT, '%') > 0 then 'PERCENT' - when norm.RESULT_UNIT is null then nvl(norm.RESULT_UNIT, 'NI') - when length(norm.RESULT_UNIT) > 11 then substr(norm.RESULT_UNIT, 1, 11) - else trim(replace(upper(norm.RESULT_UNIT), '(CALC)', '')) + when instr(lab.RESULT_UNIT, '%') > 0 then 'PERCENT' + when lab.RESULT_UNIT is null then nvl(lab.RESULT_UNIT, 'NI') + when length(lab.RESULT_UNIT) > 11 then substr(lab.RESULT_UNIT, 1, 11) + else trim(replace(upper(lab.RESULT_UNIT), '(CALC)', '')) end RESULT_UNIT -, norm.NORM_RANGE_LOW +, lab.NORM_RANGE_LOW , case - when norm.NORM_RANGE_LOW is not null and norm.NORM_RANGE_HIGH is not null then 'EQ' - when norm.NORM_RANGE_LOW is not null and norm.NORM_RANGE_HIGH is null then 'GE' - when norm.NORM_RANGE_LOW is null and norm.NORM_RANGE_HIGH is not null then 'NO' + when lab.NORM_RANGE_LOW is not null and lab.NORM_RANGE_HIGH is not null then 'EQ' + when lab.NORM_RANGE_LOW is not null and lab.NORM_RANGE_HIGH is null then 'GE' + when lab.NORM_RANGE_LOW is null and lab.NORM_RANGE_HIGH is not null then 'NO' else 'NI' end NORM_MODIFIER_LOW -, norm.NORM_RANGE_HIGH -, case nvl(nullif(norm.ABN_IND, ''), 'NI') when 'H' then 'AH' when 'L' then 'AL' when 'A' then 'AB' else 'NI' end ABN_IND +, lab.NORM_RANGE_HIGH +, case nvl(nullif(lab.ABN_IND, ''), 'NI') when 'H' then 'AH' when 'L' then 'AL' when 'A' then 'AB' else 'NI' end ABN_IND , cast(null as varchar(50)) RAW_LAB_NAME , cast(null as varchar(50)) RAW_LAB_CODE , cast(null as varchar(50)) RAW_PANEL -, case when norm.RAW_RESULT = 'T' then substr(norm.RESULT_MODIFIER, 1, 50) else to_char(norm.RESULT_NUM) end RAW_RESULT +, case when lab.RAW_RESULT = 'T' then substr(lab.RESULT_MODIFIER, 1, 50) else to_char(lab.RESULT_NUM) end RAW_RESULT , cast(null as varchar(50)) RAW_UNIT , cast(null as varchar(50)) RAW_ORDER_DEPT -, norm.RAW_FACILITY_CODE RAW_FACILITY_CODE -from lab_result_w_norm norm +, lab.RAW_FACILITY_CODE RAW_FACILITY_CODE +from lab_result_w_source lab / create index lab_result_cm_idx on lab_result_cm (PATID, ENCOUNTERID) diff --git a/i2p_tasks.py b/i2p_tasks.py index 6b60079..5b714fc 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -287,6 +287,14 @@ def requires(self) -> List[luigi.Task]: return [pcornet_init()] +class loadSpecimenSourceMap(LoadCSV): + taskName = 'SPECIMEN_SOURCE_MAP' + csvname = 'curated_data/specimen_source_map.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + class NPIDownloadConfig(luigi.Config): # The configured 'path' and 'npi' variables are used by the downloadNPI method to fetch and # store the NPPES zip file. Changes to these may require changes to the file system. From 6e678c0a12ac059cda3dc303518bdeade15ce0e4 Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 2 Jul 2018 17:10:06 -0500 Subject: [PATCH 405/507] Adding payer mapping and payer data to encounter table --- Oracle/encounter.sql | 73 ++++++-- curated_data/payer_mapping.csv | 305 +++++++++++++++++++++++++++++++++ 2 files changed, 364 insertions(+), 14 deletions(-) create mode 100644 curated_data/payer_mapping.csv diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index f581993..852fb4b 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -22,12 +22,22 @@ CREATE TABLE encounter( DRG varchar(3) NULL, DRG_TYPE varchar(2) NULL, ADMITTING_SOURCE varchar(2) NULL, + PAYER_TYPE_PRIMARY varchar(5) NULL, + PAYER_TYPE_SECONDARY varchar(5) NULL, + FACILITY_TYPE varchar(50) NULL, RAW_SITEID varchar (50) NULL, RAW_ENC_TYPE varchar(50) NULL, RAW_DISCHARGE_DISPOSITION varchar(50) NULL, RAW_DISCHARGE_STATUS varchar(50) NULL, RAW_DRG_TYPE varchar(50) NULL, - RAW_ADMITTING_SOURCE varchar(50) NULL + RAW_ADMITTING_SOURCE varchar(50) NULL, + RAW_FACILITY_TYPE varchar(50) NULL, + RAW_PAYER_TYPE_PRIMARY varchar(50) NULL, + RAW_PAYER_NAME_PRIMARY varchar(50) NULL, + RAW_PAYER_ID_PRIMARY varchar(50) NULL, + RAW_PAYER_TYPE_SECONDARY varchar(50) NULL, + RAW_PAYER_NAME_SECONDARY varchar(50) NULL, + RAW_PAYER_ID_SECONDARY varchar(50) NULL ) / BEGIN @@ -65,23 +75,56 @@ where rn=1; execute immediate 'create index drg_idx on drg (patient_num, encounter_num)'; --GATHER_TABLE_STATS('drg'); -insert into encounter(PATID,ENCOUNTERID,admit_date ,ADMIT_TIME , - DISCHARGE_DATE ,DISCHARGE_TIME ,PROVIDERID ,FACILITY_LOCATION - ,ENC_TYPE ,FACILITYID ,DISCHARGE_DISPOSITION , - DISCHARGE_STATUS ,DRG ,DRG_TYPE ,ADMITTING_SOURCE) -select distinct v.patient_num, v.encounter_num, +insert into encounter(PATID, ENCOUNTERID, admit_date, ADMIT_TIME, DISCHARGE_DATE, DISCHARGE_TIME, PROVIDERID + , FACILITY_LOCATION, ENC_TYPE, FACILITYID, DISCHARGE_DISPOSITION, DISCHARGE_STATUS, DRG, DRG_TYPE + , ADMITTING_SOURCE, PAYER_TYPE_PRIMARY, PAYER_TYPE_SECONDARY, FACILITY_TYPE, RAW_SITEID, RAW_ENC_TYPE + , RAW_DISCHARGE_DISPOSITION, RAW_DISCHARGE_STATUS, RAW_DRG_TYPE, RAW_ADMITTING_SOURCE, RAW_FACILITY_TYPE + , RAW_PAYER_TYPE_PRIMARY, RAW_PAYER_NAME_PRIMARY, RAW_PAYER_ID_PRIMARY, RAW_PAYER_TYPE_SECONDARY + , RAW_PAYER_NAME_SECONDARY, RAW_PAYER_ID_SECONDARY) +with payer as ( +select f.encounter_num + , f.patient_num + , pm.code payer_type_primary + ,f.tval_char raw_payer_name_primary + , SUBSTR(f.concept_cd, INSTR(f.concept_cd, ':', 1, 1) + 1) raw_payer_id_primary + , sf.tval_char raw_payer_type_primary +from i2b2fact f +join demographic d on f.patient_num = d.patid +left join blueherondata.supplemental_fact sf on f.instance_num = sf.instance_num +left join payer_mapping pm on lower(pm.payer_name) = lower(f.tval_char) and lower(pm.financial_class) = lower(sf.tval_char) +where concept_cd like 'KUH|PAYER:%' or concept_cd like 'KUMC|PAYER:%' +and sf.source_column = 'FINANCIAL_CLASS' +) +select distinct v.patient_num, + v.encounter_num, start_Date, to_char(start_Date,'HH24:MI'), end_Date, to_char(end_Date,'HH24:MI'), providerid, - 'NI' location_zip, /* See TODO above */ -(case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, - 'NI' facility_id, /* See TODO above */ - CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END, - CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END, - drg.drg, drg_type, - CASE WHEN admitting_source IS NULL THEN 'NI' ELSE admitting_source END admitting_source + 'NI' location_zip, /* See TODO above */ + (case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, + 'NI' facility_id, /* See TODO above */ + CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END, + CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END, + drg.drg, drg_type, + CASE WHEN admitting_source IS NULL THEN 'NI' ELSE admitting_source END admitting_source, + p.payer_type_primary, + null payer_type_secondary, + null facility_type, + null raw_siteid, + null raw_enc_type, + null raw_discharge_disposition, + null raw_discharge_status, + null raw_drg_type, + null raw_admitting_source, + null raw_facility_type, + p.raw_payer_type_primary, + p.raw_payer_name_primary, + p.raw_payer_id_primary, + null raw_payer_type_secondary, + null raw_payer_name_secondary, + null raw_payer_id_secondary from i2b2visit v inner join demographic d on v.patient_num=d.patid left outer join drg -- This section is bugfixed to only include 1 drg if multiple DRG types exist in a single encounter... on drg.patient_num=v.patient_num and drg.encounter_num=v.encounter_num @@ -89,7 +132,9 @@ left outer join -- Encounter type. Note that this requires a full table scan on the ontology table, so it is not particularly efficient. (select patient_num, encounter_num, inout_cd,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,2) pcori_enctype from i2b2visit v inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype - on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num; + on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num +left outer join payer p on p.patient_num = v.patient_num and p.encounter_num = v.encounter_num +; execute immediate 'create unique index encounter_pk on encounter (ENCOUNTERID)'; execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; diff --git a/curated_data/payer_mapping.csv b/curated_data/payer_mapping.csv new file mode 100644 index 0000000..9ee58bf --- /dev/null +++ b/curated_data/payer_mapping.csv @@ -0,0 +1,305 @@ +PAYER_NAME,FINANCIAL_CLASS,CODE,DESCRIPTIVE_TEXT +AETNA USHC HMO/POS A,AETNA,5,PRIVATE HEALTH INSURANCE +AETNA USHC HMO/POS B,AETNA,5,PRIVATE HEALTH INSURANCE +AETNA USHC PPO/TRAD A,AETNA,5,PRIVATE HEALTH INSURANCE +AETNA USHC PPO/TRAD B,AETNA,5,PRIVATE HEALTH INSURANCE +AGENCY,AGENCY,5,PRIVATE HEALTH INSURANCE +BLUE CROSS BLUE SHIELD,AGENCY,6,BLUE CROSS/BLUE SHIELD +CRIME VICTIMS KS,AGENCY,349,Other +CRIMES VICTIMS MO,AGENCY,349,Other +DEPT OF SOCIAL REHAB,AGENCY,349,Other +DISABILITY DETERMINATIONS,AGENCY,349,Other +MUSCULAR DYSTROPHY ASSOC,AGENCY,349,Other +ALT BCBS NEW DIRECTIONS,BCBS,6,BLUE CROSS/BLUE SHIELD +BCBS KANSAS,BCBS,6,BLUE CROSS/BLUE SHIELD +BCBS KC,BCBS,6,BLUE CROSS/BLUE SHIELD +BCBS KC ALTERNATE,BCBS,6,BLUE CROSS/BLUE SHIELD +EMPIRE PLAN/NYSHIP,BCBS,6,BLUE CROSS/BLUE SHIELD +INACTIVE NEW DIRECTIONS,BCBS,6,BLUE CROSS/BLUE SHIELD +PHP,BCBS,6,BLUE CROSS/BLUE SHIELD +UKH SP BCBS,BCBS,6,BLUE CROSS/BLUE SHIELD +BCBS KC HMO/POS A,BCBS KC,6,BLUE CROSS/BLUE SHIELD +BCBS KC HMO/POS B,BCBS KC,6,BLUE CROSS/BLUE SHIELD +BCBS KC INDEMNITY/OTHER,BCBS KC,6,BLUE CROSS/BLUE SHIELD +BCBS KC PPO/TRAD A,BCBS KC,6,BLUE CROSS/BLUE SHIELD +BCBS KC PPO/TRAD B,BCBS KC,6,BLUE CROSS/BLUE SHIELD +BCBS KS PREFERRED CARE OTHER,BCBS KC,6,BLUE CROSS/BLUE SHIELD +BCBS KS HMO/POS A ,BCBS KS,6,BLUE CROSS/BLUE SHIELD +BCBS KS HMO/POS B,BCBS KS,6,BLUE CROSS/BLUE SHIELD +BCBS KS PPO/TRAD A,BCBS KS,6,BLUE CROSS/BLUE SHIELD +BCBS KS PPO/TRAD B,BCBS KS,6,BLUE CROSS/BLUE SHIELD +BLUE CROSS KANSAS,BLUE CROSS KANSAS,6,BLUE CROSS/BLUE SHIELD +BLUE SHIELD,Blue Shield,6,BLUE CROSS/BLUE SHIELD +CIGNA HMO/POS A,CIGNA,5,PRIVATE HEALTH INSURANCE +CIGNA HMO/POS B,CIGNA,5,PRIVATE HEALTH INSURANCE +CIGNA PPO/TRAD A,CIGNA,5,PRIVATE HEALTH INSURANCE +CIGNA PPO/TRAD B,CIGNA,5,PRIVATE HEALTH INSURANCE +COVENTRY/PRINCIPAL HMO/POS A,COVENTRY,5,PRIVATE HEALTH INSURANCE +COVENTRY/PRINCIPAL HMO/POS B,COVENTRY,5,PRIVATE HEALTH INSURANCE +COVENTRY/PRINCIPAL PPO A,COVENTRY,5,PRIVATE HEALTH INSURANCE +COVENTRY/PRINCIPAL PPO B,COVENTRY,5,PRIVATE HEALTH INSURANCE +AETNA,Commercial,5,PRIVATE HEALTH INSURANCE +AHA-HEALTHCARE PREFERRED,Commercial,5,PRIVATE HEALTH INSURANCE +ALLEGIANCE BENEFIT PLAN MGMT INC,Commercial,5,PRIVATE HEALTH INSURANCE +ALLSTATE INSURANCE CO,Commercial,5,PRIVATE HEALTH INSURANCE +ALT COMPSYCH,Commercial,5,PRIVATE HEALTH INSURANCE +ALT COVENTRY MHNET ,Commercial,5,PRIVATE HEALTH INSURANCE +ALT GEHA,Commercial,5,PRIVATE HEALTH INSURANCE +ALT NEW DIRECTIONS,Commercial,5,PRIVATE HEALTH INSURANCE +ALT OPTUM BEHAVIORIAL HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE +ALT VALUEOPTIONS,Commercial,5,PRIVATE HEALTH INSURANCE +AMBETTER,Commercial,5,PRIVATE HEALTH INSURANCE +AMERICAN CONTINENTAL INS,Commercial,5,PRIVATE HEALTH INSURANCE +AMERICAN FAMILY LIFE INSURANCE,Commercial,5,PRIVATE HEALTH INSURANCE +AMERICAN GENERAL LIFE,Commercial,5,PRIVATE HEALTH INSURANCE +AMERICAN NATIONAL LIFE,Commercial,5,PRIVATE HEALTH INSURANCE +AMERICAN REPUBLIC INS,Commercial,5,PRIVATE HEALTH INSURANCE +AMERICAN REPUBLIC INSURANCE,Commercial,5,PRIVATE HEALTH INSURANCE +AMERICAN UNITEDLIFE,Commercial,5,PRIVATE HEALTH INSURANCE +ASSURANT HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE +BAKERY & CONFECTIONARY UNION,Commercial,5,PRIVATE HEALTH INSURANCE +BANKERS LIFE & CASUALTY,Commercial,5,PRIVATE HEALTH INSURANCE +BANKERS LIFE AND CASUALTY,Commercial,5,PRIVATE HEALTH INSURANCE +BENEFIT ADMIN SYSTEM,Commercial,5,PRIVATE HEALTH INSURANCE +BENEFIT MANAGEMENT INC,Commercial,5,PRIVATE HEALTH INSURANCE +BOILERMAKERS,Commercial,5,PRIVATE HEALTH INSURANCE +CARE SUPPLEMENT,Commercial,5,PRIVATE HEALTH INSURANCE +CENTRAL RESERVE LIFE,Commercial,5,PRIVATE HEALTH INSURANCE +CENTURY HEALTH SOLUTIONS,Commercial,5,PRIVATE HEALTH INSURANCE +CERNER,Commercial,5,PRIVATE HEALTH INSURANCE +CHAMPVA,Commercial,3221,Civilian Health and Medical Program for the VA (CHAMPVA) +CHESTERFIELD RESOURCES,Commercial,5,PRIVATE HEALTH INSURANCE +CHRISTIAN FIDELITY,Commercial,5,PRIVATE HEALTH INSURANCE +CHUBB GROUP OF INSURANCE,Commercial,5,PRIVATE HEALTH INSURANCE +CIGNA,Commercial,5,PRIVATE HEALTH INSURANCE +CONTINENTAL GENERAL INS,Commercial,5,PRIVATE HEALTH INSURANCE +CONTINENTAL LIFE OF BRENTWOOD,Commercial,5,PRIVATE HEALTH INSURANCE +CORESOURCE,Commercial,5,PRIVATE HEALTH INSURANCE +CORIZON,Commercial,5,PRIVATE HEALTH INSURANCE +CORRECT CARE SOLUTIONS,Commercial,5,PRIVATE HEALTH INSURANCE +COVENTRY,Commercial,5,PRIVATE HEALTH INSURANCE +CRIME VICTIM KS,Commercial,5,PRIVATE HEALTH INSURANCE +CRIME VICTIM MO,Commercial,5,PRIVATE HEALTH INSURANCE +DISABILITY DETERM,Commercial,5,PRIVATE HEALTH INSURANCE +EMPLOYERS HEALTH NETWORK,Commercial,5,PRIVATE HEALTH INSURANCE +EQUITABLE LIFE & CASUALTY,Commercial,5,PRIVATE HEALTH INSURANCE +FARMERS INSURANCE,Commercial,5,PRIVATE HEALTH INSURANCE +FIDELITY,Commercial,5,PRIVATE HEALTH INSURANCE +FIRST HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE +FORTIS,Commercial,5,PRIVATE HEALTH INSURANCE +FRA MILICARE PLUS,Commercial,5,PRIVATE HEALTH INSURANCE +GALLAGHER BENEFIT ADMIN,Commercial,5,PRIVATE HEALTH INSURANCE +GEHA,Commercial,5,PRIVATE HEALTH INSURANCE +GEICO,Commercial,5,PRIVATE HEALTH INSURANCE +GENERIC COMMERCIAL,Commercial,5,PRIVATE HEALTH INSURANCE +GENERIC TRANSPLANT,Commercial,5,PRIVATE HEALTH INSURANCE +GPA,Commercial,5,PRIVATE HEALTH INSURANCE +HCH ADMINISTRATION,Commercial,5,PRIVATE HEALTH INSURANCE +HEALTH COST SOLUTIONS,Commercial,5,PRIVATE HEALTH INSURANCE +HEALTHLINK,Commercial,5,PRIVATE HEALTH INSURANCE +HUMANA,Commercial,5,PRIVATE HEALTH INSURANCE +IBEW LOCAL UNION 124 H&W,Commercial,5,PRIVATE HEALTH INSURANCE +INACTIVE-TRIWEST,Commercial,5,PRIVATE HEALTH INSURANCE +INDIAN HEALTH SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE +KAISER PERMANENTE,Commercial,5,PRIVATE HEALTH INSURANCE +KANSAS CITY HOSPICE,Commercial,5,PRIVATE HEALTH INSURANCE +KENDALLWOOD HOSPICE,Commercial,5,PRIVATE HEALTH INSURANCE +KU ATHLETICS,Commercial,5,PRIVATE HEALTH INSURANCE +KU RX INTERNAL,Commercial,5,PRIVATE HEALTH INSURANCE +KU RX WAM RHTEST,Commercial,5,PRIVATE HEALTH INSURANCE +LARNED STATE HOSPITAL,Commercial,5,PRIVATE HEALTH INSURANCE +LEAGUE MEDICAL,Commercial,5,PRIVATE HEALTH INSURANCE +LIFETRAC TRANSPLANT,Commercial,5,PRIVATE HEALTH INSURANCE +LINKIA,Commercial,5,PRIVATE HEALTH INSURANCE +MEDICA,Commercial,5,PRIVATE HEALTH INSURANCE +MEDICAL EXCESS TRANSPLANT,Commercial,5,PRIVATE HEALTH INSURANCE +MEDICO INSURANCE CO,Commercial,5,PRIVATE HEALTH INSURANCE +MEGA LIFE & HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE +MERITAIN HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE +MH NET,Commercial,5,PRIVATE HEALTH INSURANCE +MOAA MEDIPLUS,Commercial,5,PRIVATE HEALTH INSURANCE +MODEL MC PAYOR,Commercial,5,PRIVATE HEALTH INSURANCE +MODEL RTE PAYOR,Commercial,5,PRIVATE HEALTH INSURANCE +MULTIPLAN,Commercial,5,PRIVATE HEALTH INSURANCE +MUTUAL OF OMAHA,Commercial,5,PRIVATE HEALTH INSURANCE +NATIONAL ASSN LETTER CARRIERS,Commercial,5,PRIVATE HEALTH INSURANCE +NECA IBEW FAMILY MEDICAL PLAN,Commercial,5,PRIVATE HEALTH INSURANCE +NORTH AMERICAN INS,Commercial,5,PRIVATE HEALTH INSURANCE +OLD SURETY LIFE INSURANCE CO,Commercial,5,PRIVATE HEALTH INSURANCE +OPTUMHEALTH,Commercial,5,PRIVATE HEALTH INSURANCE +ORSCHELN INDUSTRIES,Commercial,5,PRIVATE HEALTH INSURANCE +PHYSICIANS MUTUAL,Commercial,5,PRIVATE HEALTH INSURANCE +PLAN CARE AMERICA,Commercial,5,PRIVATE HEALTH INSURANCE +PLUMBERS LOCAL 8,Commercial,5,PRIVATE HEALTH INSURANCE +PRINCIPAL FINANCIAL GROUP,Commercial,5,PRIVATE HEALTH INSURANCE +PRISON HEALTH SYSTEMS,Commercial,5,PRIVATE HEALTH INSURANCE +PYRAMID LIFE,Commercial,5,PRIVATE HEALTH INSURANCE +QUIK TRIP,Commercial,5,PRIVATE HEALTH INSURANCE +REI,Commercial,5,PRIVATE HEALTH INSURANCE +RESERVE NATIONAL,Commercial,5,PRIVATE HEALTH INSURANCE +RX ADVANCE PCS,Commercial,5,PRIVATE HEALTH INSURANCE +RX AETNA,Commercial,5,PRIVATE HEALTH INSURANCE +RX ALLWIN DATA,Commercial,5,PRIVATE HEALTH INSURANCE +RX ALPHASCRIP,Commercial,5,PRIVATE HEALTH INSURANCE +RX ARGUS HEALTH SYSTEMS,Commercial,5,PRIVATE HEALTH INSURANCE +RX CATALYST,Commercial,5,PRIVATE HEALTH INSURANCE +RX CIGNA,Commercial,5,PRIVATE HEALTH INSURANCE +RX CVS/CAREMARK,Commercial,5,PRIVATE HEALTH INSURANCE +RX EMDEON,Commercial,5,PRIVATE HEALTH INSURANCE +RX EMPLOYEE BENEFITS MANAGEMENT SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE +RX ENVISIONRX,Commercial,5,PRIVATE HEALTH INSURANCE +RX EXPRESS SCRIPTS,Commercial,5,PRIVATE HEALTH INSURANCE +RX HEALTHTRANS,Commercial,5,PRIVATE HEALTH INSURANCE +RX HUMANA,Commercial,5,PRIVATE HEALTH INSURANCE +RX LDI INTEGRATED PHARMACY SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE +RX MAGELLAN,Commercial,5,PRIVATE HEALTH INSURANCE +RX MAXOR PLUS,Commercial,5,PRIVATE HEALTH INSURANCE +RX MEDCO,Commercial,5,PRIVATE HEALTH INSURANCE +RX MEDIMPACT,Commercial,5,PRIVATE HEALTH INSURANCE +RX MEDTRAK,Commercial,5,PRIVATE HEALTH INSURANCE +RX MO MEDICAID,Commercial,5,PRIVATE HEALTH INSURANCE +RX NATIONAL PRESCRIPTION SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE +RX NETCARD SYSTEMS,Commercial,5,PRIVATE HEALTH INSURANCE +RX OPTUM RX,Commercial,5,PRIVATE HEALTH INSURANCE +RX PBM PLUS,Commercial,5,PRIVATE HEALTH INSURANCE +RX PHARMACY DATA MANAGEMENT,Commercial,5,PRIVATE HEALTH INSURANCE +RX PMSI,Commercial,5,PRIVATE HEALTH INSURANCE +RX PRIME THERAPEUTICS,Commercial,5,PRIVATE HEALTH INSURANCE +RX PROCARE,Commercial,5,PRIVATE HEALTH INSURANCE +RX RESTAT,Commercial,5,PRIVATE HEALTH INSURANCE +RX RYAN WHITE,Commercial,5,PRIVATE HEALTH INSURANCE +RX SAV-RX,Commercial,5,PRIVATE HEALTH INSURANCE +RX THERAPY FIRST,Commercial,5,PRIVATE HEALTH INSURANCE +RX TMESYS,Commercial,5,PRIVATE HEALTH INSURANCE +RX UNITEDHEALTHCARE,Commercial,5,PRIVATE HEALTH INSURANCE +RX US SCRIPT,Commercial,5,PRIVATE HEALTH INSURANCE +SANFORD HEALTH PLAN,Commercial,5,PRIVATE HEALTH INSURANCE +SEECHANGE HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE +SELMAN AND COMPANY,Commercial,5,PRIVATE HEALTH INSURANCE +SHEET METAL WORKERS NATIONAL,Commercial,5,PRIVATE HEALTH INSURANCE +STANDARD LIFE & ACCIDENT,Commercial,5,PRIVATE HEALTH INSURANCE +STAR HRG/STAR ADMIN SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE +STARMARK,Commercial,5,PRIVATE HEALTH INSURANCE +STATE FARM INS,Commercial,5,PRIVATE HEALTH INSURANCE +STATE FARM INSURANCE,Commercial,5,PRIVATE HEALTH INSURANCE +STATE OF KANSAS,Commercial,5,PRIVATE HEALTH INSURANCE +STATE OF MISSOURI,Commercial,5,PRIVATE HEALTH INSURANCE +STERLING LIFE INS,Commercial,5,PRIVATE HEALTH INSURANCE +STUDENT ASSURANCE SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE +THE EPOCH GROUP,Commercial,5,PRIVATE HEALTH INSURANCE +THREE RIVERS PROVIDER NETWORK,Commercial,5,PRIVATE HEALTH INSURANCE +THRIVENT FOR LUTHERANS,Commercial,5,PRIVATE HEALTH INSURANCE +TRANSAMERICA LIFE INS CO,Commercial,5,PRIVATE HEALTH INSURANCE +UHC,Commercial,5,PRIVATE HEALTH INSURANCE +UHC EMPIRE/NYSHIP PB ALT,Commercial,5,PRIVATE HEALTH INSURANCE +UKH SP AETNA,Commercial,5,PRIVATE HEALTH INSURANCE +UKH SP CIGNA,Commercial,5,PRIVATE HEALTH INSURANCE +UKH SP COVENTRY,Commercial,5,PRIVATE HEALTH INSURANCE +UKH SP EOB TO 835,Commercial,5,PRIVATE HEALTH INSURANCE +UKH SP GEHA,Commercial,5,PRIVATE HEALTH INSURANCE +UKH SP GENERIC ,Commercial,5,PRIVATE HEALTH INSURANCE +UKH SP HUMANA,Commercial,5,PRIVATE HEALTH INSURANCE +UKH SP OPTUM,Commercial,5,PRIVATE HEALTH INSURANCE +UKH SP UHC,Commercial,5,PRIVATE HEALTH INSURANCE +UNITED AMERICAN INS,Commercial,5,PRIVATE HEALTH INSURANCE +UPREHS,Commercial,5,PRIVATE HEALTH INSURANCE +US MARSHALL SERVICE,Commercial,5,PRIVATE HEALTH INSURANCE +USA MANAGED CARE,Commercial,5,PRIVATE HEALTH INSURANCE +USAA LIFE INS CO,Commercial,5,PRIVATE HEALTH INSURANCE +VISION,Commercial,5,PRIVATE HEALTH INSURANCE +WILSON-MCSHANE CORP,Commercial,5,PRIVATE HEALTH INSURANCE +WPPA,Commercial,5,PRIVATE HEALTH INSURANCE +GRANT/STUDY,GRANT/STUDY,5,PRIVATE HEALTH INSURANCE +HUMANA HMO A,HUMANA,5,PRIVATE HEALTH INSURANCE +HUMANA HMO B,HUMANA,5,PRIVATE HEALTH INSURANCE +HUMANA PPO A,HUMANA,5,PRIVATE HEALTH INSURANCE +HUMANA PPO B,HUMANA,5,PRIVATE HEALTH INSURANCE +FAMILY HEALTH PARTNERS,MEDICAID KS,2,MEDICAID +KS MEDICAID,MEDICAID KS,2,MEDICAID +KS MEDICAID HMO B,MEDICAID KS,2,MEDICAID +MEDICAID KS,MEDICAID KS,2,MEDICAID +MEDICAID KS HOM(102 REPLACEMEN,MEDICAID KS,2,MEDICAID +UKH SP KS MEDICAID,MEDICAID KS,2,MEDICAID +MEDICAID MO,MEDICAID MO,2,MEDICAID +MEDICAID MO HMO,MEDICAID MO,2,MEDICAID +MEDICAID OUT OF STATE,MEDICAID OUT OF STATE,25,Medicaid - Out of State +KS MEDICAID PENDING,MEDICAID PENDING,2,MEDICAID +MO MEDICAID PENDING,MEDICAID PENDING,2,MEDICAID +MEDICAID,Medicaid,2,MEDICAID +ALT MO MEDICAID,Medicaid OOS,25,Medicaid - Out of State +MEDICAID OUT OF STATE,Medicaid OOS,25,Medicaid - Out of State +MO MEDICAID,Medicaid OOS,25,Medicaid - Out of State +UKH SP OOS MEDICAID,Medicaid OOS,25,Medicaid - Out of State +ALT CENPATICO KS MCAID BEHAVIORAL,Medicaid Repl KS,2,MEDICAID +AMERIGROUP MEDICAID KS,Medicaid Repl KS,2,MEDICAID +CENTENE MEDICAID KS,Medicaid Repl KS,2,MEDICAID +MEDICAID OTHER HMO KS GENERIC,Medicaid Repl KS,2,MEDICAID +UHC MEDICAID KS,Medicaid Repl KS,2,MEDICAID +VIA CHRISTI HOPE,Medicaid Repl KS,2,MEDICAID +AETNA MEDICAID OOS,Medicaid Repl OOS,25,Medicaid - Out of State +ALT CENPATICO MO MCAID BEHAVIORAL,Medicaid Repl OOS,25,Medicaid - Out of State +COVENTRY MEDICAID OOS,Medicaid Repl OOS,25,Medicaid - Out of State +HOME STATE HEALTH PLAN,Medicaid Repl OOS,25,Medicaid - Out of State +MEDICAID OTHER HMO OOS GENERIC,Medicaid Repl OOS,25,Medicaid - Out of State +MISSOURI CARE,Medicaid Repl OOS,25,Medicaid - Out of State +UHC MEDICAID MO,Medicaid Repl OOS,25,Medicaid - Out of State +ALT MEDICARE,Medicare,1,MEDICARE +ALT MEDICARE-NORIDIAN,Medicare,1,MEDICARE +CARE,Medicare,1,MEDICARE +DMERC,Medicare,1,MEDICARE +MEDICARE,Medicare,1,MEDICARE +MEDICARE HMO,Medicare,1,MEDICARE +MEDICARE HMO B,Medicare,1,MEDICARE +MEDICARE PPO A,Medicare,1,MEDICARE +MEDICARE RAILROAD,Medicare,1,MEDICARE +UKH SP MEDICARE,Medicare,1,MEDICARE +AETNA MEDICARE,Medicare Repl,1,MEDICARE +ALLWELL (OON),Medicare Repl,1,MEDICARE +ALT COVENTRY ADVANTRA MHNET,Medicare Repl,1,MEDICARE +BCBS KC MEDICARE,Medicare Repl,1,MEDICARE +CARE IMPROVEMENT PLUS,Medicare Repl,1,MEDICARE +CHILDREN'S MERCY HOSPITAL,Medicare Repl,1,MEDICARE +CIGNA MEDICARE,Medicare Repl,1,MEDICARE +CONSOLIDATED BILLING,Medicare Repl,1,MEDICARE +COVENTRY MEDICARE,Medicare Repl,1,MEDICARE +HUMANA MEDICARE,Medicare Repl,1,MEDICARE +MEDICARE REPL GENERIC,Medicare Repl,1,MEDICARE +TODAY'S OPTIONS,Medicare Repl,1,MEDICARE +UHC MEDICARE,Medicare Repl,1,MEDICARE +AUTO/LIABILITY,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE +GLOBAL TRANSPLANTS,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE +KUH EMPL SUPPLEMENTAL PLAN,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE +LIABILITY OTHER (NOT AUTO/WC),OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE +LITIGATION/LEGAL ,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE +OTHER/COMMERCIAL HMO/POS A,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE +OTHER/COMMERCIAL HMO/POS B,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE +OTHER/COMMERCIAL PPO/TRAD A,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE +OTHER/COMMERCIAL PPO/TRAD B,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE +AGENCY,Other,349,Other +METRO CARE,Other,349,Other +NORTHLAND CARE,Other,349,Other +PROJECT EAGLE,Other,349,Other +STUDY,Other,349,Other +WYJO CARE,Other,349,Other +ELECTIVE PACKAGE RATE,Self-pay,81,Self-pay +PATIENT REQUEST-DON'T BILL INS,Self-pay,81,Self-pay +SELF PAY,Self-pay,81,Self-pay +HEALTH NET FEDERAL SERVICES,Tricare,3113,TRICARE Standard - Fee For Service +TRICARE,Tricare,3113,TRICARE Standard - Fee For Service +TRICARE HMO,Tricare,3111,TRICARE Prime?HMO +TRICARE PPO/TRAD,Tricare,3112,TRICARE Extra?PPO +UKH SP TRICARE,Tricare,3113,TRICARE Standard - Fee For Service +UNITED HEALTH/METRA HMO/POS A,UNITED HEALTH,5,PRIVATE HEALTH INSURANCE +UNITED HEALTH/METRA HMO/POS B,UNITED HEALTH,5,PRIVATE HEALTH INSURANCE +UNITED HEALTH/METRA PPO/TRAD A,UNITED HEALTH,5,PRIVATE HEALTH INSURANCE +UNITED HEALTH/METRA PPO/TRAD B,UNITED HEALTH,5,PRIVATE HEALTH INSURANCE +CHAMPVA,VA,3221,Civilian Health and Medical Program for the VA (CHAMPVA) +TRIWEST,VA,32,Department of Veterans Affairs +UKH SP CHAMPVA/VA,VA,32,Department of Veterans Affairs +VA,VA,32,Department of Veterans Affairs +VETERANS ADMINISTRATION,VA,32,Department of Veterans Affairs +GALLAGHER BASSETT,Worker's Comp,95,Worker's Compensation +GENERIC WORKERS COMP,Worker's Comp,95,Worker's Compensation +MS WORKERS COMP,Worker's Comp,95,Worker's Compensation +WORKERS COMP,Worker's Comp,95,Worker's Compensation +WORKMAN'S COMPENSATION,Worker's Comp,95,Worker's Compensation From 6f57286d0aedfc90159c2b4d68c0bd500e753315 Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 2 Jul 2018 17:15:09 -0500 Subject: [PATCH 406/507] Adding specimen source map. --- curated_data/specimen_source_map.csv | 492 +++++++++++++++++++++++++++ 1 file changed, 492 insertions(+) create mode 100644 curated_data/specimen_source_map.csv diff --git a/curated_data/specimen_source_map.csv b/curated_data/specimen_source_map.csv new file mode 100644 index 0000000..25ec95f --- /dev/null +++ b/curated_data/specimen_source_map.csv @@ -0,0 +1,492 @@ +SPECIMEN_SOURCE_NAME,CODE +,^BPU +,^BPU.AUTOLOGOUS +,^BPU.PLATELET_PHERESIS +,^CANCER +,^EGG_DONOR +,^EMBRYO +,^FATHER +Fetus,^FETUS +,^GUARDIAN_OR_LEGALLY_AUTHORIZED_REPRESENTATIVE +,^MOTHER +,^MUSHROOM_SPECIMEN +,^PATIENT +,^PLANT_SPECIMEN +,^POPULATION +,^STEM_CELL_PRODUCT +,^TICK +,{SETTING} +,ABDOMEN.FNA +Abscess,ABSCESS +"Adrenal Gland,Left",ADRENAL_GLAND +"Adrenal Gland,Right",ADRENAL_GLAND +,AIR +Amniotic Fluid,AMNIO_FLD +,AMNIO_FLD_CELLS +,AMNIO_FLD_CVS +Anus,ANAL +,ANOGENITAL +,ANORECTAL +,ANORECTAL_ISOLATE +,ANORECTAL_STOOL +Aorta,AORTA.ROOT +,ASP +,B_CELLS+MONOCYTES +,BAL +Bartholin Cyst,BARTHOLIN_CYST +,BBL +Biliary Fluid,BIL_FLD +Blood,BLD +Blood Clot,BLD +"Blood, Arterial",BLD +"Blood,Line Draw",BLD +"Blood,Peripheral",BLD +,BLD_MC +,BLD.BUFFY_COAT +,BLD.DOT +,BLD.POS_GROWTH +,BLD_BONE_MAR +,BLD_BONE_MAR^DONOR +,BLD_TISS +,BLD_TISS_SAL +,BLD_TISS^DONOR +,BLD_URINE +,BLD^BPU +,BLD^CONTROL +,BLD^DONOR +,BLD^FATHER +,BLD^FETUS +,BLD^NEWBORN +,BLD^PATIENT +,BLD+INHL_GAS +,BLDA +,BLDA+INHL_GAS +,BLDC +,BLDC^FETUS +,BLDC+INHL_GAS +,BLDCO +,BLDCOA +,BLDCOMV +,BLDCOV +,BLDCRRT +,BLDMV +,BLDMV+INHL_GAS +,BLDP +,BLDV +,BLDV+INHL_GAS +,BODY_FLD +,BODY_FLD.SPUN +,BODY_FLD^FETUS +,BODY_FLD+SER_PLAS +Bone,BONE +Bone Specimen,BONE +,BONE_MAR +Bone Marrow,BONE_MARROW +,BONE^DONOR +Brain,BRAIN +Breast,BREAST +"Breast,Left",BREAST +"Breast,Right",BREAST +"Breast Cyst,Left",BREAST +"Breast Cyst,Right",BREAST +"Nipple Smear,Left",BREAST +"Nipple Smear,Right",BREAST +,BREAST_CANCER_SPECIMEN +,BREAST_TUMOR +,BREAST.DUCTAL_LAVAGE +,BREAST.FNA +Bronchial Alveolar Lavage,BRONCHIAL +"Bronchial Alveolar Lavage,LLL",BRONCHIAL +"Bronchial Alveolar Lavage,LUL",BRONCHIAL +"Bronchial Alveolar Lavage,RLL",BRONCHIAL +"Bronchial Alveolar Lavage,RML",BRONCHIAL +"Bronchial Alveolar Lavage,RUL",BRONCHIAL +Bronchial Wash(Specify Site),BRONCHIAL +Bronchial Washing LLL,BRONCHIAL +Bronchial Washing LUL,BRONCHIAL +Bronchial Washing RLL,BRONCHIAL +Bronchial Washing RML,BRONCHIAL +Bronchial Washing RUL,BRONCHIAL +Bronchus,BRONCHIAL +Bronchial Brush (Specify Site),BRONCHIAL_BRUSH +"Bronchial Brush,LLL",BRONCHIAL_BRUSH +"Bronchial Brush,LUL",BRONCHIAL_BRUSH +"Bronchial Brush,RLL",BRONCHIAL_BRUSH +"Bronchial Brush,RML",BRONCHIAL_BRUSH +"Bronchial Brush,RUL",BRONCHIAL_BRUSH +Buccal Mucosa,BUCCAL +,BUCCAL_SMEAR +,BURN +,BURSA_OF_FABRICIUS +,CALCULUS +,CANCER_SPECIMEN +,CELLS.XXX +Cervix,CERVIX +,CHEESE +,CNJT +,CNL +,COL +Colon,COLON +,COLORECTAL_CANCER_SPECIMEN +,CONTACT_LENS +,CORONARY_SINUS +,CRN +,CSF +,CSF.SPUN +,CTP +,CURRENT_SAMPLE +,CVM +,CVS +,CVX +,CVX_VAG +,DAIRY_PRODUCT +,DEEP_TISSUE.FNA +,DENTIN +Dialysate,DIAL_FLD +,DIAL_FLD_PRT +,DIAL_FLD_PRT+SER_PLAS +,DIAL_FLD.SPUN +,DIAL_FLD+SER_PLAS +,DOSE +,DRAIN +,DUCTUS_ARTERIOSUS +,DUOD_FLD +,DUOD_FLD_GAST_FLD +Ear,EAR +Ear Middle,EAR +Middle Ear,EAR +Auditory Canal,EAR +Auditory Canal Internal,EAR +Tympanic Cavity,EAR +Eustachian Tube,EAR +,EAR_FLUID +,EGG +,EGGYLK +,ENDOCERVICAL_BRUSH +Endometrial,ENDOMET +Endocardium,ENDOMYOCARDIUM +,ENVIR +,ENVIRONMENTAL_SPECIMEN +Esophageal Brush,ESOPHAGEAL_BRUSH +,EXHL_GAS +,EXTRACELLULAR_FLD +,EXUDATE +Eye,EYE +Cornea,EYE +Corneal Scrapings,EYE +Conjunctiva,EYE +Eye Drainage,EYE +Eyelid,EYE +Contact Lens Solution,EYE +,FACILITY +,FEATHER +,FEED +,FIBROBLASTS +,FIBROBLASTS^CONTROL +,FLU.NONBIOLOGICAL +,FOOD +,GAS +,GAST_FLD +,GENITAL +,GENITAL_FLD +,GENITAL_LOC +,GENITAL_MUC +,GI_CNT +,GLUCOSE_METER_DEVICE +Hair,HAIR +,HEART.ATRIUM.LEFT +,HEART.ATRIUM.RIGHT +,HEART.ATRIUM.RIGHT.HIGH +,HEART.ATRIUM.RIGHT.LOW +,HEART.ATRIUM.RIGHT.MID +,HEART.VENTRICLE.LEFT +,HEART.VENTRICLE.RIGHT +,HEART.VENTRICLE.RIGHT.OUTFLOW_TRACT +,HEMATOPOIETIC_PROGENITOR_CELLS^BPU +,HYPERAL_SOLUTION +,IMPLANT_DEVICE +,INDEX_CASE^COMPARISON_CASE +,INGESTA +,INHL_GAS +,INITIAL_SAMPLE +,INTRATHECAL_SPACE +,INTRAVASCULAR_SPACE.XXX +,ISOLATE +,ISOLATE.MENINGITIS +,ISOLATE.PNEUMONIA +,ISOLATE.UTI +,ISOLATE+SER +Kidney,KIDNEY +Kidney Basin Fluid,KIDNEY +"Kidney, Left",KIDNEY +"Kidney, Right",KIDNEY +,KIDNEY.FNA +"Line,Arterial",LINE +,LITTER +Liver,LIVER +,LIVER.FNA +,LIVER^FETUS +Lung,LUNG +Lung LLL,LUNG +Lung LUL,LUNG +Lung RLL,LUNG +Lung RML,LUNG +Lung RUL,LUNG +Pulmonic,LUNG +,LUNG_TISS +,LUNG.FNA +Lymph Node (Specify),LYMPH_NODE +,LYMPH_NODE.FNA +,LYMPHOBLASTS +,MECONIUM +,MESORECTUM +Breast Milk,MILK +,MLK.RAW +,MMLK +Mouth,MOUTH +Oral Cavity,MOUTH +Oropharynx,MOUTH +Gingiva,MOUTH +Uvula,MOUTH +Mouth Washing,MOUTH +Mandible,MOUTH +Tongue,MOUTH +Teeth/Tooth,MOUTH +Muscle,MUSCLE +,NAIL +,NASAL_FLUID +,NBS_CARD +,NECK_MASS.FNA +,NIPPLE_DISCHARGE +Nose,NOSE +Nasal,NOSE +Nasal Wash,NOSE +Nasopharyngeal Aspirate,NOSE +Nasopharyngeal Swab,NOSE +Turbinate,NOSE +,NOSE_TRAC +,NPH +,OCULAR_FLD +Ovary,OVARY +Pancreas,PANCREAS +"Pancreas,Body",PANCREAS +"Pancreas,Head",PANCREAS +"Pancreas,Neck",PANCREAS +"Pancreas,Tail",PANCREAS +Pancreatic Fluid,PANCREAS +Pancreatic Pseudocyst,PANCREAS +,PANCREAS.FNA +,PAROTID.FNA +,PELVIS.FNA +Penis,PENIS +Penile Discharge,PENIS +Penile Lesion,PENIS +Pericardial Fluid,PERICARD_FLD +Pericardial,PERICARD_FLD +Pericardium,PERICARD_FLD +,PERICARD_FLD.SPUN +,PERICARD_FLD+SER_PLAS +Peritoneal Fluid,PERITON_FLD +Ascitic Fluid,PERITON_FLD +,PERITON_FLD.SPUN +,PERITON_FLD+SER_PLAS +Peritoneal Cavity,PERITONEUM +Peritoneal Dialysate Fluid,PERITONEUM +Peritoneal Lavage,PERITONEUM +Peritoneum,PERITONEUM +Pharynx,PHARYNX +Placenta,PLACENT +,PLANT +,PLAS +,PLAS.CFDNA +,PLAS_BLD +,PLAS_RBC +,PLAS^DONOR +,PLAS+CSF +,PLAS+URINE +,PLASA +,PLASV +Platelets,PLATELETS +Pleural Cavity,PLEURA +"Pleural,Left",PLEURA +"Pleural,Right",PLEURA +Pleural Fluid,PLR_FLD +,PLR_FLD.SPUN +,PLR_FLD^FETUS +,POC +,PPP +,PPP_BLD +,PPP^CONTROL +,PPP^FETUS +,PPP^POOL +,PREPUTIAL_WASH +Prostate,PROSTATE +,PROSTATE_CANCER.XXX +,PROSTATE_TUMOR +,PROSTATE.FNA +,PROSTATIC_FLD +,PROVIDER +,PRP +,PRP^CONTROL +,PRP^DONOR +,PULMONARY_ARTERY.LEFT +,PULMONARY_ARTERY.MAIN +,PULMONARY_ARTERY.RIGHT +,PULMONARY_WEDGE +Pus,PUS +Rbc,RBC +,RBC^BPU +,RBC^CONTROL +,RBC^DONOR +,RBC^FETUS +,RBC^PATIENT +,RBCCO +Rectum,RECTUM +Rectal Abscess,RECTUM +Rectal Swab,RECTUM +,REFERENCE_LAB_TEST +,REPORT +,REPOSITORY +,RESPIRATORY +,RESPIRATORY.LOWER +,RESPIRATORY.UPPER +,RETIC +,RH_IMMUNE_GLOBULIN +Saliva,SALIVA +,SALIVARY_GLAND.FNA +Semen,SEMEN +,SEMEN+CVM +,SEMIN_PLAS +,SER +,SER_BLD +,SER_PLAS +,SER_PLAS.ULTRACENTRIFUGATE +,SER_PLAS_BLD +,SER_PLAS_BLDV +,SER_PLAS_URINE +,SER_PLAS^BPU +,SER_PLAS^DONOR +,SER_PLAS^FETUS +,SER_PLAS^NORMAL_CONTROL +,SER_PLAS+CSF +,SER_PLAS+PLR_FLD +,SER_PLAS+STOOL +,SER_PLAS+SYNV_FLD +,SER^CONTROL +,SER^DONOR +,SER+BLD +,SER+CSF +,SER+PLAS +,SER+RBC^CONTROL +,SER+SALIVA +Sinus,SINUS +Skin,SKIN +Skin Biopsy,SKIN +Skin Scrapings,SKIN +"Skin Swab,Burn Unit",SKIN +"Skin Swab,Marrow Transplant",SKIN +,SKIN_MELANOMA +,SMALL_INTES_FIXED +,SMPLS +,SOFT_TISSUE.FNA +,SPECIMEN +,SPERMATOZOA +,SPIRAL_COLON +Spleen,SPLEEN +,SPTC +,SPTT +Sputum,SPUTUM +"Sputum,Cystic Fibrosis",SPUTUM +,SPUTUM_BRONCHIAL +,SPUTUM_GAST_FLD +Stomach,STOMACH +Gastric,STOMACH +Gastric Aspirate,STOMACH +Pylorus,STOMACH +,STOMACH_CANCER +Feces,STOOL +,STOOL.DRIED +,STOOL.WET +,STUDY +,SUBMANDIBULAR.FNA +,SUPERFICIAL_TISSUE.FNA +Sweat,SWEAT +Synovial Fluid,SYNV_FLD +Synovial,SYNV_FLD +Synovium,SYNV_FLD +,SYNV_FLD.SPUN +,TEAR +Testis,TESTIS +Esophageal,THRT +Esophagus,THRT +Epiglottis,THRT +Thyroid,THYROID +"Thyroid,Left",THYROID +"Thyroid,Right",THYROID +,THYROID.FNA +,TISS +,TISS_FAT +,TISS_FIXED +,TISS.FNA +,TISS^CONTROL +,TISS^FETUS +Tissue,TISSUE +Tissue Specimen,TISSUE +,TISSUE_CHIP +,TISSUE_CORE +,TLGI_TSMI +,TPN +Trachea,TRAC +Tracheal Aspirate,TRAC +Tracheostomy Site,TRAC +Transtracheal Aspirate,TRAC +,TRACHEAL_SWAB +,TROPHOBLASTS +,TSMI +,TUMOR +Ulcer,ULC +,UNK_SUB +Urethra,URETHRA +Urinary Bladder,URINARY_BLADDER +Urine,URINE +Urine Catheter,URINE +Urine Straight Catheter,URINE +"Urine, Foley Catheter",URINE +"Urine, Outpatient",URINE +"Urine,Clean Catch",URINE +"Urine,Nephrostomy",URINE +"Urine,Renal Transplant",URINE +"Urine,Suprapubic",URINE +,URINE_SED +,URINE^FETUS +,URINE+SER_PLAS +Uterus,UTERUS +Myometrium,UTERUS +Vaginal,VAG +Vaginal-Cpt,VAG +Vaginal/Cervical,VAG +Rectal Vaginal,VAG+RECTUM +,VENA_CAVA.INFERIOR +,VENA_CAVA.SUPERIOR +Vitreous Fluid,VITR_FLD +,VITR_FLD.SPUN +,VOMITUS +,VP_SHUNT +,WATER +,WBC +,WBC.DNA+PLAS.CFDNA +,WBC^CONTROL +,WHEY +Wound,WOUND +,WOUND.DEEP +,WOUND.SHLW +,XXX +,XXX_MC +,XXX.BODY_FLUID +,XXX.SWAB +,XXX.TISSUE +,XXX^DONOR +,NI +,UN +Other (Specify),OT +Other Specimen,OT From 3abf678ff76b04d531cff1e89ac895c7c7d6fa6a Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 2 Jul 2018 17:53:43 -0500 Subject: [PATCH 407/507] Adding payer map load op to i2p_task and tweaking map name. --- Oracle/encounter.sql | 2 +- curated_data/{payer_mapping.csv => payer_map.csv} | 0 i2p_tasks.py | 13 +++++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) rename curated_data/{payer_mapping.csv => payer_map.csv} (100%) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 852fb4b..975faf5 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -91,7 +91,7 @@ select f.encounter_num from i2b2fact f join demographic d on f.patient_num = d.patid left join blueherondata.supplemental_fact sf on f.instance_num = sf.instance_num -left join payer_mapping pm on lower(pm.payer_name) = lower(f.tval_char) and lower(pm.financial_class) = lower(sf.tval_char) +left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) and lower(pm.financial_class) = lower(sf.tval_char) where concept_cd like 'KUH|PAYER:%' or concept_cd like 'KUMC|PAYER:%' and sf.source_column = 'FINANCIAL_CLASS' ) diff --git a/curated_data/payer_mapping.csv b/curated_data/payer_map.csv similarity index 100% rename from curated_data/payer_mapping.csv rename to curated_data/payer_map.csv diff --git a/i2p_tasks.py b/i2p_tasks.py index 6b60079..c42a847 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -260,6 +260,19 @@ def requires(self) -> List[luigi.Task]: return [encounter()] +class loadPayerMap(LoadCSV): + taskName = 'PAYER_MAP' + # payer_map.csv is a combination of the CDM spec's payer_type spreadsheet + # with payer_name and financial_class values from epic's clarity_epm table. + # payer_name and financial_class are used together to determine the CDM's + # payer type code. + # TODO: incorporte IDX. + csvname = 'curated_data/payer_map.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + class loadLabNormal(LoadCSV): taskName = 'LABNORMAL' csvname = 'curated_data/labnormal.csv' From 4038b7a8002c135429241dfbe001116bda1bc18a Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 5 Jul 2018 09:34:15 -0500 Subject: [PATCH 408/507] Matching payer concepts codes to changes made in Heron. --- Oracle/encounter.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 975faf5..45f703d 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -82,6 +82,7 @@ insert into encounter(PATID, ENCOUNTERID, admit_date, ADMIT_TIME, DISCHARGE_DATE , RAW_PAYER_TYPE_PRIMARY, RAW_PAYER_NAME_PRIMARY, RAW_PAYER_ID_PRIMARY, RAW_PAYER_TYPE_SECONDARY , RAW_PAYER_NAME_SECONDARY, RAW_PAYER_ID_SECONDARY) with payer as ( +-- TODO: Nulls and IDX. select f.encounter_num , f.patient_num , pm.code payer_type_primary @@ -92,7 +93,7 @@ from i2b2fact f join demographic d on f.patient_num = d.patid left join blueherondata.supplemental_fact sf on f.instance_num = sf.instance_num left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) and lower(pm.financial_class) = lower(sf.tval_char) -where concept_cd like 'KUH|PAYER:%' or concept_cd like 'KUMC|PAYER:%' +where concept_cd like 'O2|PRIMARYPAYER:%' or concept_cd like 'IDX|PRIMARYPAYER:%' and sf.source_column = 'FINANCIAL_CLASS' ) select distinct v.patient_num, From a0969809b114eaaa073ec1e1bf7d39faba3ce62a Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 5 Jul 2018 09:48:17 -0500 Subject: [PATCH 409/507] Adding comments to clarify the construction of specimen source map. --- i2p_tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/i2p_tasks.py b/i2p_tasks.py index 5b714fc..38796dd 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -289,6 +289,8 @@ def requires(self) -> List[luigi.Task]: class loadSpecimenSourceMap(LoadCSV): taskName = 'SPECIMEN_SOURCE_MAP' + # specimen_source_map.csv matches values in the CDM spec's specimen_source spreadsheet + # to specimen type values from Epic's order_proc table. csvname = 'curated_data/specimen_source_map.csv' def requires(self) -> List[luigi.Task]: From 8e6b3ced9e637f85f774672a465b565033d870d4 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 5 Jul 2018 17:36:07 -0500 Subject: [PATCH 410/507] Changing concept codes to match work in Heron. --- Oracle/encounter.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 45f703d..0c7f5cc 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -93,7 +93,7 @@ from i2b2fact f join demographic d on f.patient_num = d.patid left join blueherondata.supplemental_fact sf on f.instance_num = sf.instance_num left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) and lower(pm.financial_class) = lower(sf.tval_char) -where concept_cd like 'O2|PRIMARYPAYER:%' or concept_cd like 'IDX|PRIMARYPAYER:%' +where concept_cd like 'O2|PAYER_PRIMARY:%' or concept_cd like 'IDX|PAYER_PRIMARY:%' and sf.source_column = 'FINANCIAL_CLASS' ) select distinct v.patient_num, From 36fcb196751b19c714770e599e703bc1c53bedcb Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 11 Jul 2018 17:28:51 -0500 Subject: [PATCH 411/507] First draft of changes to support CDM 4.1 fields. --- Oracle/dispensing.sql | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index a538110..c0e666a 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -14,7 +14,13 @@ CREATE TABLE dispensing( NDC varchar (11) NOT NULL, DISPENSE_SUP number(18) NULL, DISPENSE_AMT number(18) NULL, - RAW_NDC varchar (50) NULL + DISPENSE_DOSE_DISP number(18) NULL, + DISPENSE_DOSE_DISP_UNIT varchar(50) NULL, + DISPENSE_ROUTE varchar(50) NULL, + RAW_NDC varchar (50) NULL, + RAW_DISPENSE_DOSE_DISP varchar(50) NULL, + RAW_DISPENSE_DOSE_DISP_UNIT varchar(50) NULL, + RAW_DISPENSE_ROUTE varchar(50) NULL ) / @@ -91,7 +97,13 @@ insert into dispensing ( ,NDC --using pcornet_med pcori_ndc - new column! ,DISPENSE_SUP ---- modifier nval_num ,DISPENSE_AMT -- modifier nval_num --- ,RAW_NDC + ,DISPENSE_DOSE_DISP + ,DISPENSE_DOSE_DISP_UNIT + ,DISPENSE_ROUTE + ,RAW_NDC + ,RAW_DISPENSE_DOSE_DISP + ,RAW_DISPENSE_DOSE_DISP_UNIT + ,RAW_DISPENSE_ROUTE ) /* Below is the Cycle 2 fix for populating the DISPENSING table */ with disp_status as ( @@ -124,7 +136,14 @@ select distinct st.start_date dispense_date, replace(st.concept_cd, 'NDC:', '') ndc, -- TODO: Generalize this for other sites. ds.nval_num dispense_sup, - qt.nval_num dispense_amt + qt.nval_num dispense_amt, + to_number(sf1.tval_char) dispense_dose_disp, + sf2.tval_char dispense_dose_disp_unit, + sf3.tval_char dispense_route, + null raw_ndc, + sf1.tval_char raw_dispense_dose_disp, + sf2.tval_char raw_dispense_dose_disp_unit, + sf3.tval_char raw_dispense_route from disp_status st left outer join disp_quantity qt on st.patient_num=qt.patient_num @@ -138,6 +157,18 @@ left outer join disp_supply ds and st.concept_cd=ds.concept_cd and st.instance_num=ds.instance_num and st.start_date=ds.start_date +left outer join blueherondata.supplemental_fact sf1 + on st.encounter_num = sf1.encounter_num + and st.instance_num = sf1.instance_num + and sf1.source_column = 'DISCRETE_DOSE' +left outer join blueherondata.supplemental_fact sf2 + on st.encounter_num = sf2.encounter_num + and st.instance_num = sf2.instance_num + and sf2.source_column = 'DOSE_UNITS' +left outer join blueherondata.supplemental_fact sf3 + on st.encounter_num = sf3.encounter_num + and st.instance_num = sf3.instance_num + and sf3.source_column = 'ADMIN_ROUTE' ; /* NOTE: The original SCILHS transformation is below. From b41ffc9f287833136c80e14a40bf73391a6c87d0 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 12 Jul 2018 10:15:21 -0500 Subject: [PATCH 412/507] Adding raw language spoken to demographic table. --- Oracle/demographic.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql index b1ac6d9..beab9ae 100644 --- a/Oracle/demographic.sql +++ b/Oracle/demographic.sql @@ -21,7 +21,8 @@ CREATE TABLE demographic( RAW_SEXUAL_ORIENTATION varchar(50) NULL, RAW_GENDER_IDENTITY varchar(50) NULL, RAW_HISPANIC varchar(50) NULL, - RAW_RACE varchar(50) NULL + RAW_RACE varchar(50) NULL, + RAW_PAT_PREF_LANGUAGE_SPOKEN varchar(50) NULL ) / create or replace procedure PCORNetDemographic as @@ -202,7 +203,8 @@ using ( ) l on (d.patid = l.patient_num) when matched then update -set d.PAT_PREF_LANGUAGE_SPOKEN = l.code; +set d.PAT_PREF_LANGUAGE_SPOKEN = l.code, +d.RAW_PAT_PREF_LANGUAGE_SPOKEN = l.language_cd; execute immediate 'create unique index demographic_pk on demographic (PATID)'; GATHER_TABLE_STATS('DEMOGRAPHIC'); From 8339b1c22d094207e66cbec0bc67c5ef57b2ba5c Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 12 Jul 2018 16:28:00 -0500 Subject: [PATCH 413/507] Adding present on admission (POA) to diagnosis. Heron ETL provides POA as a valueless fact when true, granular POA codes are not recorded. This work will need to be revisited after updates are made in Heron. --- Oracle/diagnosis.sql | 49 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index d303d0c..15e2652 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -18,11 +18,13 @@ CREATE TABLE diagnosis( DX_SOURCE varchar(2) NOT NULL, DX_ORIGIN varchar(2) NULL, PDX varchar(2) NULL, + DX_POA varchar(2) NULL, RAW_DX varchar(50) NULL, RAW_DX_TYPE varchar(50) NULL, RAW_DX_SOURCE varchar(50) NULL, RAW_ORIGDX varchar(50) NULL, - RAW_PDX varchar(50) NULL + RAW_PDX varchar(50) NULL, + RAW_DX_POA varchar(50) NULL ) / @@ -88,6 +90,24 @@ CREATE TABLE ORIGINFACT ( C_FULLNAME VARCHAR2(700) NOT NULL ) / + +BEGIN +PMN_DROPSQL('DROP TABLE poafact'); +END; +/ + +CREATE TABLE POAFACT ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + POASOURCE VARCHAR2(50) NULL, + RAWPOASOURCE VARCHAR2(50) NULL, + C_FULLNAME VARCHAR2(700) NOT NULL + ) +/ + create or replace procedure PCORNetDiagnosis as begin @@ -95,11 +115,13 @@ PMN_DROPSQL('drop index diagnosis_idx'); PMN_DROPSQL('drop index sourcefact_idx'); PMN_DROPSQL('drop index pdxfact_idx'); PMN_DROPSQL('drop index originfact_idx'); +PMN_DROPSQL('drop index poafact_idx'); execute immediate 'truncate table diagnosis'; execute immediate 'truncate table sourcefact'; execute immediate 'truncate table pdxfact'; execute immediate 'truncate table originfact'; +execute immediate 'truncate table poafact'; insert into sourcefact select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname @@ -131,7 +153,17 @@ insert into originfact --CDM 3.1 addition execute immediate 'create index originfact_idx on originfact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('ORIGINFACT'); -insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, dx_origin, pdx) +insert into poafact + select patient_num, encounter_num, provider_id, concept_cd, start_date, 'Y' poasource, 'Yes' rawpoasource, dxsource.c_fullname + from i2b2fact factline + inner join ENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd = dxsource.c_basecode + and dxsource.c_fullname like '\PCORI_MOD\DX_ORIGIN\BI\DX|BILL:POA\%'; + +execute immediate 'create index poafact_idx on poafact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('POAFACT'); + +insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, dx_origin, pdx, dx_poa, raw_dx_poa) /* KUMC started billing with ICD10 on Oct 1, 2015. */ with icd10_transition as ( select date '2015-10-01' as cutoff from dual @@ -213,7 +245,11 @@ select distinct factline.patient_num, factline.encounter_num encounterid, enc_ty nvl(SUBSTR(originsource,INSTR(originsource, ':')+1,2),'NI') dx_origin, CASE WHEN enc_type in ('EI', 'IP', 'IS', 'OS') THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, ':')+1,2),'NI') - ELSE null END PDX + ELSE null END PDX, + CASE WHEN enc_type in ('EI', 'IP') + THEN nvl(SUBSTR(poasource,INSTR(poasource, ':')+1,2),'UN') + ELSE null END DX_POA + , rawpoasource RAW_DX_POA from diag_fact_cutoff_filter factline inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num left outer join sourcefact @@ -234,6 +270,13 @@ and factline.encounter_num=originfact.encounter_num and factline.provider_id=originfact.provider_id and factline.concept_cd=originfact.concept_cd and factline.start_date=originfact.start_Date +left outer join poafact +on factline.patient_num=poafact.patient_num +and factline.encounter_num=poafact.encounter_num +and factline.provider_id=poafact.provider_id +and factline.concept_cd=poafact.concept_cd +and factline.start_date=poafact.start_Date + where (sourcefact.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\%' or sourcefact.c_fullname is null) -- order by enc.admit_date desc ; From b2722c8e543f064b821cf8ecb60232aa9bbfd44e Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 13 Jul 2018 15:51:11 -0500 Subject: [PATCH 414/507] Adding PRN flag to prescribing. Plus 4.1 columns set to null for other tables. --- Oracle/lab_result_cm.sql | 2 +- Oracle/prescribing.sql | 31 ++++++++++++++++++++++++++++--- Oracle/pro_cm.sql | 33 +++++++++++++++++++++++---------- Oracle/procedures.sql | 4 +++- 4 files changed, 55 insertions(+), 15 deletions(-) diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 7143581..8ae525a 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -136,7 +136,6 @@ create table lab_result_cm as select distinct cast(lab.LAB_RESULT_CM_ID as varchar(19)) LAB_RESULT_CM_ID , cast(lab.PATID as varchar(50)) PATID , cast(lab.ENCOUNTERID as varchar(50)) ENCOUNTERID -, case when lab.LAB_NAME like 'LAB_NAME%' then substr(lab.LAB_NAME, 10, 10) else 'UN' end LAB_NAME , lab.SPECIMEN_SOURCE , nvl(lab.LAB_LOINC, 'NI') LAB_LOINC , 'NI' PRIORITY @@ -149,6 +148,7 @@ select distinct cast(lab.LAB_RESULT_CM_ID as varchar(19)) LAB_RESULT_CM_ID , lab.RESULT_DATE , to_char(lab.RESULT_DATE, 'HH24:MI') RESULT_TIME , 'NI' RESULT_QUAL +, cast(null as varchar(50)) RESULT_SNOMED , case when lab.RAW_RESULT = 'N' then lab.RESULT_NUM else null end RESULT_NUM , case when lab.RAW_RESULT = 'N' then (case nvl(nullif(lab.RESULT_MODIFIER, ''),'NI') when 'E' then 'EQ' when 'NE' then 'OT' when 'L' then 'LT' when 'LE' then 'LE' when 'G' then 'GT' when 'GE' then 'GE' else 'NI' end) else 'TX' end RESULT_MODIFIER , case diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 135b49a..c188292 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -210,6 +210,22 @@ left join ) dose on dose.instance_num = rx.instance_num and dose.concept_cd = rx.concept_cd / +create table prescribing_w_prn as +select rx.* +, nvl(prn.tval_char, 'NO') rx_prn_flag +from prescribing_w_dose rx +left join + (select instance_num + , concept_cd + , 'YES' as tval_char + from blueherondata.observation_fact + where modifier_cd = 'MedObs:PRN' + /* aka: + select c_basecode from pcornet_med code + where code.c_fullname like '\PCORI_MOD\RX_BASIS\PR\02\MedObs:PRN\' */ + ) prn on prn.instance_num = rx.instance_num and prn.concept_cd = rx.concept_cd +/ + create table prescribing as select rx.prescribingid , rx.patient_num patid @@ -222,20 +238,29 @@ select rx.prescribingid , rx.rx_dose_ordered , rx.rx_dose_ordered_unit , rx.rx_quantity -, 'NI' rx_quantity_unit +, 'NI' rx_dose_form , rx.rx_refills , rx.rx_days_supply , rx.rx_frequency +, rx.rx_prn_flag +, null rx_route , decode(rx.modifier_cd, 'MedObs:Inpatient', '01', 'MedObs:Outpatient', '02') rx_basis , rx.rxnorm_cui +, null rx_source +, null rx_dispense_as_written , rx.raw_rx_med_name , cast(null as varchar(50)) raw_rx_frequency +, rx.raw_rxnorm_cui , cast(null as varchar(50)) raw_rx_quantity , cast(null as varchar(50)) raw_rx_ndc -, rx.raw_rxnorm_cui +, cast(null as varchar(50)) raw_rx_dose_ordered +, cast(null as varchar(50)) raw_rx_dose_ordered_unit +, cast(null as varchar(50)) raw_rx_route +, cast(null as varchar(50)) raw_rx_refills + /* ISSUE: HERON should have an actual order time. idea: store real difference between order date start data, possibly using the update date */ -from prescribing_w_dose rx +from prescribing_w_prn rx / create index prescribing_idx on prescribing (PATID, ENCOUNTERID) diff --git a/Oracle/pro_cm.sql b/Oracle/pro_cm.sql index e15701a..1cab3d5 100644 --- a/Oracle/pro_cm.sql +++ b/Oracle/pro_cm.sql @@ -9,17 +9,30 @@ END; CREATE TABLE pro_cm( PRO_CM_ID varchar(19) primary key, PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NULL, - PRO_ITEM varchar (20) NOT NULL, - PRO_LOINC varchar (10) NULL, + ENCOUNTERID varchar(50) NULL, PRO_DATE date NOT NULL, - PRO_TIME varchar (5) NULL, - PRO_RESPONSE int NOT NULL, - PRO_METHOD varchar (2) NULL, - PRO_MODE varchar (2) NULL, - PRO_CAT varchar (2) NULL, - RAW_PRO_CODE varchar (50) NULL, - RAW_PRO_RESPONSE varchar (50) NULL + PRO_TIME varchar(5) NULL, + PRO_TYPE varchar(2) NULL, + PRO_ITEM_NAME varchar(50) NULL, + PRO_ITEM_LOINC varchar(10) NULL, + PRO_RESPONSE_TEXT varchar(50) NULL, + PRO_RESPONSE_NUM number(8) NOT NULL, + PRO_METHOD varchar(2) NULL, + PRO_MODE varchar(2) NULL, + PRO_CAT varchar(2) NULL, + PRO_ITEM_VERSION varchar(50) NULL, + PRO_MEASURE_NAME varchar(50) NULL, + PRO_MEASURE_SEQ varchar(50) NULL, + PRO_MEASURE_SCORE varchar(50) NULL, + PRO_MEASURE_THETA number(8) NULL, + PRO_MEASURE_SCALED_TSCORE number(8) NULL, + PRO_MEASURE_STANDARD_ERROR number(8) NULL, + PRO_MEASURE_COUNT_SCORED number(8) NULL, + PRO_MEASURE_LOINC varchar(10) NULL, + PRO_MEASURE_VERSION varchar(50) NULL, + PRO_ITEM_FULLNAME varchar(50) NULL, + PRO_ITEM_TEXT varchar(50) NULL, + PRO_MEASURE_FULLNAME varchar(50) NULL ) / diff --git a/Oracle/procedures.sql b/Oracle/procedures.sql index f121c3a..fb6b36f 100644 --- a/Oracle/procedures.sql +++ b/Oracle/procedures.sql @@ -17,8 +17,10 @@ CREATE TABLE procedures( PX varchar(11) NOT NULL, PX_TYPE varchar(2) NOT NULL, PX_SOURCE varchar(2) NULL, + PPX varchar(2) NULL, RAW_PX varchar(50) NULL, - RAW_PX_TYPE varchar(50) NULL + RAW_PX_TYPE varchar(50) NULL, + RAW_PPX varchar(50) NULL ) / BEGIN From 50ebac23d48d7abbe07f42359bb26c0ab46882c2 Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 16 Jul 2018 09:07:09 -0500 Subject: [PATCH 415/507] Removing unnecessary obs fact meds check. --- Oracle/pcornet_init.sql | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 0f8af64..edbd2fa 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -16,11 +16,7 @@ select case when qty = 0 then 1/0 else 1 end inout_cd_populated from ( -- Make sure the RXNorm mapping table exists select rxcui from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id where 1=0 / --- Make sure the observation fact medication table is populated -select case when qty > 0 then 1 else 1/0 end obs_fact_meds_populated from ( - select count(*) qty from observation_fact_meds - ) -/ + create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS BEGIN DBMS_STATS.GATHER_TABLE_STATS ( From d7d5c4345e1ad775e0740dea9e5e7769415ca18f Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 16 Jul 2018 09:17:15 -0500 Subject: [PATCH 416/507] Correcting syntax errors in lab and prescribing scripts. --- Oracle/lab_result_cm.sql | 1 + Oracle/prescribing.sql | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 8ae525a..dffaf0c 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -30,6 +30,7 @@ END; BEGIN PMN_DROPSQL('drop table lab_result_w_source'); END; +/ BEGIN PMN_DROPSQL('DROP SEQUENCE lab_result_cm_seq'); diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index c188292..01ebbaa 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -243,11 +243,11 @@ select rx.prescribingid , rx.rx_days_supply , rx.rx_frequency , rx.rx_prn_flag -, null rx_route +, cast(null as varchar(50)) rx_route , decode(rx.modifier_cd, 'MedObs:Inpatient', '01', 'MedObs:Outpatient', '02') rx_basis , rx.rxnorm_cui -, null rx_source -, null rx_dispense_as_written +, cast(null as varchar(2)) rx_source +, cast(null as varchar(2)) rx_dispense_as_written , rx.raw_rx_med_name , cast(null as varchar(50)) raw_rx_frequency , rx.raw_rxnorm_cui From e3ba479ab3d7daba7544df635786d68650d5f64b Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 16 Jul 2018 09:27:24 -0500 Subject: [PATCH 417/507] Adding drop statement for new prn table to prescribing script. --- Oracle/prescribing.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 01ebbaa..2a0e631 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -39,6 +39,10 @@ PMN_DROPSQL('drop table prescribing_w_dose'); END; / BEGIN +PMN_DROPSQL('drop table prescribing_w_prn'); +END; +/ +BEGIN PMN_DROPSQL('DROP sequence prescribing_seq'); END; / From 29e9f50d8cd09a6091ace5f35c4f92ff4070273b Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 16 Jul 2018 17:01:10 -0500 Subject: [PATCH 418/507] Route map for dispensing and prescrbing plus various bug fixes. --- Oracle/dispensing.sql | 3 +- Oracle/harvest.sql | 6 +- Oracle/prescribing.sql | 30 ++-- curated_data/harvest_local.csv | 2 +- curated_data/route_map.csv | 208 +++++++++++++++++++++++++++ curated_data/specimen_source_map.csv | 6 +- i2p_tasks.py | 12 ++ 7 files changed, 251 insertions(+), 16 deletions(-) create mode 100644 curated_data/route_map.csv diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index c0e666a..a3c29e2 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -139,7 +139,7 @@ select distinct qt.nval_num dispense_amt, to_number(sf1.tval_char) dispense_dose_disp, sf2.tval_char dispense_dose_disp_unit, - sf3.tval_char dispense_route, + rm.code dispense_route, null raw_ndc, sf1.tval_char raw_dispense_dose_disp, sf2.tval_char raw_dispense_dose_disp_unit, @@ -169,6 +169,7 @@ left outer join blueherondata.supplemental_fact sf3 on st.encounter_num = sf3.encounter_num and st.instance_num = sf3.instance_num and sf3.source_column = 'ADMIN_ROUTE' +left outer join route_map rm on lower(sf3.tval_char) = lower(rm.route_name) ; /* NOTE: The original SCILHS transformation is below. diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index 36f02ec..9761395 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -35,7 +35,7 @@ CREATE TABLE harvest( PRO_DATE_MGMT varchar(2) NULL, DEATH_DATE_MGTM varchar(2) NULL, MEDADMIN_START_DATE_MGMT varchar(2) NULL, - MEDADMIN_END_DATE_MGMT varchar(2) NULL, + MEDADMIN_STOP_DATE_MGMT varchar(2) NULL, OBSCLIN_DATE_MGMT varchar(2) NULL, OBSGEN_DATE_MGMT varchar(2) NULL, REFRESH_DEMOGRAPHIC_DATE date NULL, @@ -66,7 +66,7 @@ execute immediate 'truncate table harvest'; INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, - ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, DEATH_DATE_MGTM, MEDADMIN_START_DATE_MGMT, MEDADMIN_END_DATE_MGMT, + ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, DEATH_DATE_MGTM, MEDADMIN_START_DATE_MGMT, MEDADMIN_STOP_DATE_MGMT, OBSCLIN_DATE_MGMT, OBSGEN_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, @@ -75,7 +75,7 @@ INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.PX_DATE_MGMT, hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, - hl.DEATH_DATE_MGMT, hl.MEDADMIN_START_DATE_MGMT, hl.MEDADMIN_END_DATE_MGMT, hl.OBSCLIN_DATE_MGMT, hl.OBSGEN_DATE_MGMT, + hl.DEATH_DATE_MGMT, hl.MEDADMIN_START_DATE_MGMT, hl.MEDADMIN_STOP_DATE_MGMT, hl.OBSCLIN_DATE_MGMT, hl.OBSGEN_DATE_MGMT, case when (select records from cdm_status where task = 'demographic') > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, case when (select records from cdm_status where task = 'enrollment') > 0 then current_date else null end REFRESH_ENROLLMENT_DATE, case when (select records from cdm_status where task = 'encounter') > 0 then current_date else null end REFRESH_ENCOUNTER_DATE, diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 2a0e631..e5ba839 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -201,8 +201,8 @@ left join create table prescribing_w_dose as select rx.* -, units_cd rx_dose_ordered -, nval_num rx_dose_ordered_unit +, nval_num rx_dose_ordered +, units_cd rx_dose_ordered_unit from prescribing_w_basis rx left join (select instance_num @@ -230,6 +230,20 @@ left join ) prn on prn.instance_num = rx.instance_num and prn.concept_cd = rx.concept_cd / +create table prescribing_w_route as +select rx.* +, nvl(rm.code, 'NI') rx_route +, rt.tval_char raw_rx_route +from prescribing_w_prn rx +left join + (select instance_num + , tval_char + from blueherondata.supplemental_fact + where source_column = 'PRESCRIBING_ROUTE' + ) rt on rt.instance_num = rx.instance_num +left join route_map rm on lower(rt.tval_char) = lower(rm.route_name) +/ + create table prescribing as select rx.prescribingid , rx.patient_num patid @@ -247,24 +261,24 @@ select rx.prescribingid , rx.rx_days_supply , rx.rx_frequency , rx.rx_prn_flag -, cast(null as varchar(50)) rx_route +, rx.rx_route , decode(rx.modifier_cd, 'MedObs:Inpatient', '01', 'MedObs:Outpatient', '02') rx_basis , rx.rxnorm_cui -, cast(null as varchar(2)) rx_source +, 'OD' rx_source , cast(null as varchar(2)) rx_dispense_as_written , rx.raw_rx_med_name , cast(null as varchar(50)) raw_rx_frequency , rx.raw_rxnorm_cui , cast(null as varchar(50)) raw_rx_quantity , cast(null as varchar(50)) raw_rx_ndc -, cast(null as varchar(50)) raw_rx_dose_ordered -, cast(null as varchar(50)) raw_rx_dose_ordered_unit -, cast(null as varchar(50)) raw_rx_route +, rx.rx_dose_ordered raw_rx_dose_ordered +, rx.rx_dose_ordered_unit raw_rx_dose_ordered_unit +, rx.raw_rx_route , cast(null as varchar(50)) raw_rx_refills /* ISSUE: HERON should have an actual order time. idea: store real difference between order date start data, possibly using the update date */ -from prescribing_w_prn rx +from prescribing_w_route rx / create index prescribing_idx on prescribing (PATID, ENCOUNTERID) diff --git a/curated_data/harvest_local.csv b/curated_data/harvest_local.csv index 035f41a..cd46ff5 100644 --- a/curated_data/harvest_local.csv +++ b/curated_data/harvest_local.csv @@ -1,2 +1,2 @@ -"DATAMART_CLAIMS","DATAMART_EHR","BIRTH_DATE_MGMT","ENR_START_DATE_MGMT","ENR_END_DATE_MGMT","ADMIT_DATE_MGMT","DISCHARGE_DATE_MGMT","PX_DATE_MGMT","RX_ORDER_DATE_MGMT","RX_START_DATE_MGMT","RX_END_DATE_MGMT","DISPENSE_DATE_MGMT","LAB_ORDER_DATE_MGMT","SPECIMEN_DATE_MGMT","RESULT_DATE_MGMT","MEASURE_DATE_MGMT","ONSET_DATE_MGMT","REPORT_DATE_MGMT","RESOLVE_DATE_MGMT","PRO_DATE_MGMT","DEATH_DATE_MGMT","MEDADMIN_START_DATE_MGMT","MEDADMIN_END_DATE_MGMT","OBSCLIN_DATE_MGMT","OBSGEN_DATE_MGMT" +"DATAMART_CLAIMS","DATAMART_EHR","BIRTH_DATE_MGMT","ENR_START_DATE_MGMT","ENR_END_DATE_MGMT","ADMIT_DATE_MGMT","DISCHARGE_DATE_MGMT","PX_DATE_MGMT","RX_ORDER_DATE_MGMT","RX_START_DATE_MGMT","RX_END_DATE_MGMT","DISPENSE_DATE_MGMT","LAB_ORDER_DATE_MGMT","SPECIMEN_DATE_MGMT","RESULT_DATE_MGMT","MEASURE_DATE_MGMT","ONSET_DATE_MGMT","REPORT_DATE_MGMT","RESOLVE_DATE_MGMT","PRO_DATE_MGMT","DEATH_DATE_MGMT","MEDADMIN_START_DATE_MGMT","MEDADMIN_STOP_DATE_MGMT","OBSCLIN_DATE_MGMT","OBSGEN_DATE_MGMT" "01","02","03","03","03","03","03","03","03","03","04","03","03","03","03","03","03","03","03","03","03","03","03","03","03" diff --git a/curated_data/route_map.csv b/curated_data/route_map.csv new file mode 100644 index 0000000..0100508 --- /dev/null +++ b/curated_data/route_map.csv @@ -0,0 +1,208 @@ +ROUTE_NAME,CODE +adductor canal block,OT +Apheresis,EXTRACORPOREAL +Apply externally,OT +Arterial,INTRA_ARTERIAL +Base of the eyelashes,OT +Bladder Instillation,OT +Block,OT +Both Ears,OTIC +Both Eyes,INTRAOCULAR +Buccal,BUCCAL +Catheter Sheath,OT +Caudal Block,CAUDAL +Central,OT +central line irrigation,OT +Cervical,OT +Chest Tube,OT +Combination,OT +CONT CAUDAL INFUSION,CAUDAL +CONT INTRAARTER INF,OT +CONT INTRAOSSEOUS IF,OT +CONT INTRATHECAL INF,INTRATHECAL +CONT NEBULIZATION,OT +CONT SUBCUTAN INFUSI,SUBCUTANEOUS +Contin. Intraosseous Infusion,OT +Contin. Intrathecal Infusion,INTRATHECAL +Contin. Subcutaneous Infusion,SUBCUTANEOUS +Contin.Intra-Arterial Infusion,INTRA_ARTERIAL +continuous adductor canal block,OT +Continuous Caudal Infusion,CAUDAL +CONTINUOUS EPIDURAL,EPIDURAL +CONTINUOUS INFILTRAT,OT +continuous infraclavicular,OT +CONTINUOUS INFUSION,OT +continuous interscalene,OT +Continuous Intraperitoneal Infusion,INTRAPERITONEAL +Continuous IV Infusion,INTRAVENOUS +Continuous Nebulization,OT +CRRT,OT +cytapheresis,OT +deal,OT +dental,DENTAL +Dialysis,OT +Does not apply,UN +Each Nostril,NASAL +Endotracheal,OT +Epidural,EPIDURAL +EXTRACORPOREAL,EXTRACORPOREAL +Feeding,OT +Feeding Tube,OT +Flush,OT +Gastrostomy Tube,GASTROSTOMY +GU irrigant,OT +Gums,GINGIVAL +HAND BULB NEBULIZER,OT +Hemodialysis,OT +hemodialysis port injection,OT +IM/PO DISCRETIONARY,OT +Implant,OT +In Vitro,OT +Infiltration,OT +infraclavicular,OT +Inhalation,OT +Injection,OT +Insufflation,OT +Intercatheter,OT +interscalene,INTRAMUSCULAR +Intra-amniotic,INTRAAMNIOTIC +Intra-arterial,INTRA_ARTERIAL +Intra-articular,INTRA_ARTICULAR +INTRABURSAL,INTRABURSAL +Intracameral,INTRACAMERAL +Intracardiac,INTRACARDIAC +Intra-catheter,OT +Intracavernosal,INTRACAVERNOUS +INTRACAVITY,BODY_CAVITY +Intradermal,INTRADERMAL +INTRADETRUSOR,OT +Intraductal,INTRADUCTAL +Intralesional,INTRALESIONAL +Intra-Lesional,INTRALESIONAL +INTRALUMBAR,INTRASPINAL +Intra-lymphatic,INTRALYMPHATIC +Intramuscular,INTRAMUSCULAR +Intranasal,INTRASINAL +Intraocular,INTRAOCULAR +intraocular injection,INTRAOCULAR +intraocular irrigation,INTRAOCULAR +Intraosseous,INTRAOSSEOUS +INTRAPERICARDIAL,INTRAPERICARDIAL +Intraperitoneal,INTRAPERITONEAL +Intrapleural,INTRAPLEURAL +Intraspinal,INTRASPINAL +Intrasynovial,INTRASYNOVIAL +Intrathecal,INTRATHECAL +Intrathoracic,INTRATHORACIC +Intratracheal,INTRATRACHEAL +intratympanic,INTRATYMPANIC +INTRA-URETHRAL,URETHRAL +Intrauterine,INTRAUTERINE +INTRAVARICEAL,INTRAVENOUS +Intravenous,INTRAVENOUS +intravenous push,OT +INTRAVENTRICULAR,INTRACEREBROVENTRICULAR +Intravesical,INTRAVESICAL +intravesical irrigation,INTRAVESICAL +INTRAVITREAL,INTRAVITREAL +Iontophoresis,OT +IPPB,OT +Irrigation,OT +JUXTASCLERAL,OT +Laryngotracheal,OT +Left Ear,OTIC +Left Eye,INTRAOCULAR +Local Infiltration,OT +local intranasal application,OT +Misc.(Non-Drug; Combo Route),OT +MISCELLANEOUS,OT +Mouth/Throat,ORAL +Mucous Membrane,OT +Nares,NASAL +Nasal,NASAL +Nasogastric,NASOGASTRIC +nasogastric tube,NASOGASTRIC +Nebulization,OT +NEBULIZATION -UNSPEC,OT +NOT APPLICABLE,UN +O2 AEROSOLIZATION,OT +Ommaya Reservior,OT +One Nostril,NASAL +Ophthalmic,OPHTHALMIC +Oral,ORAL +ORAL/IV,OT +ORAL/RECTAL/IM,OT +ORAL/RECTAL/IV,OT +Osteochondrial,OT +Otic,OTIC +PEG Tube,PERCUTANEOUS +PEG Tube,PERCUTANEOUS +PEG-J tube,PERCUTANEOUS +Per Corpak Tube,OT +Per Dobhoff Tube,NASOGASTRIC +Per G Tube,GASTROSTOMY +Per J Tube,JEJUNOSTOMY +Per NG tube,NASOGASTRIC +Per OG Tube,OROGASTRIC +Percutaneous,PERCUTANEOUS +Perfusion,OT +Perianal,OT +periarticular,OT +Peri-articular,PERIARTICULAR +PERIBULBAR,PERIBULBAR +Pericapsular,OT +Perineal,OT +perineural injection,PERINEURAL +Periodontal,PERIODONTAL +Periodontal pocket,OT +Peripheral,OT +Peripheral Arterial Line,OT +peripheral nerve block,OT +Peripheral Nerve Cath,OT +Peritoneal catheter,OT +Pump Refill,OT +Rectal,RECTAL +RECTAL/ORAL DISCRET,OT +RETROBULBAR,RETROBULBAR +Retrobulbular,RETROBULBAR +Right Ear,OTIC +Right Eye,INTRAOCULAR +scalp,OT +Scratch,OT +SEE ADMIN INSTRUCTIONS,UN +Service,OT +Sheath,OT +Sub lesionally,SUBLESIONAL +SUBCONJUCTIVAL,SUBCONJUNCTIVAL +Subconjunctival,SUBCONJUNCTIVAL +Subcutaneous,SUBCUTANEOUS +Subdermal,OT +Subgingival-Local,SUBGINGIVAL +Sublesional,SUBLESIONAL +Sublesionally,SUBLESIONAL +Sublingual,SUBLINGUAL +Submucosal anal canal,OT +SUBMUCOSAL INJ,SUBMUCOSAL +Submucosal Injection,SUBMUCOSAL +SubQ Pump,OT +subretinal,OT +SUB-TENON,OT +Swish & Spit,ORAL +Swish & Swallow,ORAL +TENDON SHEATH INJ.,OT +Tendon Sheath Injection,OT +Test,OT +Topical,TOPICAL +Tracheal Tube,OT +Transdermal,TRANSDERMAL +Translingual,OT +TRANSTRACHEAL,TRANSTRACHEAL +TRANSURETHRAL,TRANSURETHRAL +Tube,OT +Umbilical Artery Cath,OT +Unknown,UN +Urethral,URETHRAL +urinary catheter irrigation,OT +Vaginal,VAGINAL +venous line irrigation,OT +Wound irrigation,OT diff --git a/curated_data/specimen_source_map.csv b/curated_data/specimen_source_map.csv index 25ec95f..392e3f2 100644 --- a/curated_data/specimen_source_map.csv +++ b/curated_data/specimen_source_map.csv @@ -38,9 +38,9 @@ Bartholin Cyst,BARTHOLIN_CYST Biliary Fluid,BIL_FLD Blood,BLD Blood Clot,BLD -"Blood, Arterial",BLD +"Blood, Arterial",BLDA "Blood,Line Draw",BLD -"Blood,Peripheral",BLD +"Blood,Peripheral",BLDP ,BLD_MC ,BLD.BUFFY_COAT ,BLD.DOT @@ -177,7 +177,7 @@ Corneal Scrapings,EYE Conjunctiva,EYE Eye Drainage,EYE Eyelid,EYE -Contact Lens Solution,EYE +Contact Lens Solution,OT ,FACILITY ,FEATHER ,FEED diff --git a/i2p_tasks.py b/i2p_tasks.py index 5771f26..3cc140c 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -310,9 +310,20 @@ def requires(self) -> List[luigi.Task]: return [pcornet_init()] +class loadRouteMap(LoadCSV): + taskName = 'ROUTE_MAP' + # route_map.csv matches values in the CDM spec's _route spreadsheet + # to route values from Epic's zc_admin_route table. + csvname = 'curated_data/route_map.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + class NPIDownloadConfig(luigi.Config): # The configured 'path' and 'npi' variables are used by the downloadNPI method to fetch and # store the NPPES zip file. Changes to these may require changes to the file system. + # TODO: Update code to discover the npi_csv automatically. dl_path = StrParam(description='Path where the NPPES zip file will be stored and unzipped.') extract_path = StrParam(description='Path where the extract') npi_csv = StrParam(description='CSV file in the NPPES zip that contains NPI data.') @@ -322,6 +333,7 @@ class NPIDownloadConfig(luigi.Config): # The configured 'col' and 'ct' variables reflect the layout of the NPI data file. # The extracNPI method uses these values to parse the NPI data file. # Changes to these may require code changes. + # Complete overkill making these configurable. Consider reverting to hard coded values. taxonomy_col = StrParam(description='Header for the taxonomy columns in the NPI data file.') switch_col = StrParam(description='Header for the switch columns in the NPI data file.') npi_col = StrParam(description='Header for the NPI column in the NPI data file.') From 2e90574f56294e81da1299cab6dc75bea9a7cab3 Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 17 Jul 2018 17:35:13 -0500 Subject: [PATCH 419/507] Adding unit map, plus join on dispensing and prescribing. --- Oracle/dispensing.sql | 59 ++- Oracle/prescribing.sql | 36 +- curated_data/unit_map.csv | 816 ++++++++++++++++++++++++++++++++++++++ i2p_tasks.py | 10 + 4 files changed, 879 insertions(+), 42 deletions(-) create mode 100644 curated_data/unit_map.csv diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index a3c29e2..86b2050 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -130,6 +130,31 @@ with disp_status as ( on ibf.modifier_cd=pnm.c_basecode where pnm.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\%' ) +, disp_dose as ( + select encounter_num + , instance_num + , to_number(tval_char) dose + from blueherondata.supplemental_fact + where source_column = 'DISCRETE_DOSE' +) +, disp_unit as ( + select sf.encounter_num + , sf.instance_num + , nvl(um.code, 'OT') code + , sf.tval_char + from blueherondata.supplemental_fact sf + left join unit_map um on sf.tval_char = um.unit_name + where sf.source_column = 'DOSE_UNITS' +) +, disp_route as ( + select sf.encounter_num + , sf.instance_num + , rm.code + , sf.tval_char + from blueherondata.supplemental_fact sf + left join route_map rm on lower(sf.tval_char) = lower(rm.route_name) + where sf.source_column = 'ADMIN_ROUTE' +) select distinct st.patient_num patid, null prescribingid, @@ -137,13 +162,13 @@ select distinct replace(st.concept_cd, 'NDC:', '') ndc, -- TODO: Generalize this for other sites. ds.nval_num dispense_sup, qt.nval_num dispense_amt, - to_number(sf1.tval_char) dispense_dose_disp, - sf2.tval_char dispense_dose_disp_unit, - rm.code dispense_route, + dd.dose dispense_dose_disp, + du.code dispense_dose_disp_unit, + dr.code dispense_route, null raw_ndc, - sf1.tval_char raw_dispense_dose_disp, - sf2.tval_char raw_dispense_dose_disp_unit, - sf3.tval_char raw_dispense_route + dd.dose raw_dispense_dose_disp, + du.tval_char raw_dispense_dose_disp_unit, + dr.tval_char raw_dispense_route from disp_status st left outer join disp_quantity qt on st.patient_num=qt.patient_num @@ -157,19 +182,15 @@ left outer join disp_supply ds and st.concept_cd=ds.concept_cd and st.instance_num=ds.instance_num and st.start_date=ds.start_date -left outer join blueherondata.supplemental_fact sf1 - on st.encounter_num = sf1.encounter_num - and st.instance_num = sf1.instance_num - and sf1.source_column = 'DISCRETE_DOSE' -left outer join blueherondata.supplemental_fact sf2 - on st.encounter_num = sf2.encounter_num - and st.instance_num = sf2.instance_num - and sf2.source_column = 'DOSE_UNITS' -left outer join blueherondata.supplemental_fact sf3 - on st.encounter_num = sf3.encounter_num - and st.instance_num = sf3.instance_num - and sf3.source_column = 'ADMIN_ROUTE' -left outer join route_map rm on lower(sf3.tval_char) = lower(rm.route_name) +left outer join disp_dose dd + on st.encounter_num = dd.encounter_num + and st.instance_num = dd.instance_num +left outer join disp_unit du + on st.encounter_num = du.encounter_num + and st.instance_num = du.instance_num +left outer join disp_route dr + on st.encounter_num = dr.encounter_num + and st.instance_num = dr.instance_num ; /* NOTE: The original SCILHS transformation is below. diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index e5ba839..cdf8471 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -77,15 +77,20 @@ from dual create table prescribing_key as select cast(prescribing_seq.nextval as varchar(19)) prescribingid , instance_num -, cast(patient_num as varchar(50)) patient_num -, cast(encounter_num as varchar(50)) encounter_num -, provider_id +, cast(patient_num as varchar(50)) patid +, cast(encounter_num as varchar(50)) encounterid +, provider_id rx_providerid , start_date , end_date , concept_cd , modifier_cd +, case when trim(translate(tval_char, '0123456789.', ' ')) is null then tval_char else null end rx_dose_ordered +, case when trim(translate(tval_char, '0123456789.', ' ')) is null then um.code else null end rx_dose_order +, tval_char raw_rx_dose_ordered +, units_cd raw_rx_dose_ordered_unit from blueherondata.observation_fact rx join encounter en on rx.encounter_num = en.encounterid +join unit_map um on rx.units_cd = um.unit_name where rx.modifier_cd in ('MedObs:Inpatient', 'MedObs:Outpatient') / @@ -199,25 +204,10 @@ left join ) basis on basis.instance_num = rx.instance_num and basis.concept_cd = rx.concept_cd / -create table prescribing_w_dose as -select rx.* -, nval_num rx_dose_ordered -, units_cd rx_dose_ordered_unit -from prescribing_w_basis rx -left join - (select instance_num - , concept_cd - , case when units_cd = 'mcg' then 'ug' when units_cd = 'l' then 'ml' else units_cd end units_cd - , case when units_cd = 'l' then nval_num * 1000 else nval_num end nval_num - from blueherondata.observation_fact - where modifier_cd in ('MedObs:Dose|mg', 'MedObs:Dose|meq', 'MedObs:Dose|l') - ) dose on dose.instance_num = rx.instance_num and dose.concept_cd = rx.concept_cd -/ - create table prescribing_w_prn as select rx.* , nvl(prn.tval_char, 'NO') rx_prn_flag -from prescribing_w_dose rx +from prescribing_w_basis rx left join (select instance_num , concept_cd @@ -246,14 +236,14 @@ left join route_map rm on lower(rt.tval_char) = lower(rm.route_name) create table prescribing as select rx.prescribingid -, rx.patient_num patid -, rx.encounter_num encounterid -, rx.provider_id rx_providerid +, rx.patid +, rx.encounterid +, rx.rx_providerid , trunc(rx.start_date) rx_order_date , to_char(rx.start_date, 'HH24:MI') rx_order_time , trunc(rx.start_date) rx_start_date , trunc(rx.end_date) rx_end_date -, rx.rx_dose_ordered +, to_number(rx.rx_dose_ordered) rx_dose_ordered , rx.rx_dose_ordered_unit , rx.rx_quantity , 'NI' rx_dose_form diff --git a/curated_data/unit_map.csv b/curated_data/unit_map.csv new file mode 100644 index 0000000..5b9d3f2 --- /dev/null +++ b/curated_data/unit_map.csv @@ -0,0 +1,816 @@ +CODE,UNIT_NAME +%,% +%{abnormal}, +%{activity}, +%{aggregation}, +%{at_60_min}, +%{bacteria}, +%{basal_activity}, +%{baseline}, +%{binding}, +%{blockade}, +%{blocked}, +%{bound}, +%{breakdown}, +%{cells}, +%{deficient}, +%{dose}, +%{excretion}, +%{Hb}, +%{hemolysis}, +%{index}, +%{inhibition}, +%{loss_AChR}, +%{loss}, +%{lysis}, +%{normal}, +%{penetration}, +%{pooled_plasma}, +%{positive}, +%{RBCs}, +%{reactive}, +%{recovery}, +%{reference}, +%{relative}, +%{residual}, +%{saturation}, +%{total}, +%{uptake}, +%{viable}, +%{vol}, +%{WBCs}, +/(12.h), +/[arb'U], +/[HPF], +/[IU], +/[LPF], +/{entity}, +/{OIF}, +/10*10, +/10*12, +/10*12{RBCs}, +/10*3, +/10*3{RBCs}, +/10*4{RBCs}, +/10*6, +/10*9, +/100, +/100{cells}, +/100{neutrophils}, +/100{spermatozoa}, +/100{WBCs}, +/a, +/cm[H2O], +/d, +/dL, +/g, +/g{creat}, +/g{Hb}, +/g{tot_nit}, +/g{tot_prot}, +/g{wet_tis}, +/h, +/kg, +/kg{body_wt}, +/L, +/m2, +/m3, +/mg, +/min, +/mL, +/mm, +/mmol{creat}, +/mo, +/s, +/U, +/uL, +/wk, +[APL'U], +[APL'U]/mL, +[arb'U], +[arb'U]/mL, +[AU], +[bdsk'U], +[beth'U], +[CFU], +[CFU]/L, +[CFU]/mL, +[Ch], +[cin_i], +[degF], +[dr_av], +[drp], +[foz_us], +[ft_i], +[gal_us], +[GPL'U], +[GPL'U]/mL, +[HPF], +[in_i], +[in_i'H2O], +[IU],Int'l Units +[IU]/(2.h), +[IU]/(24.h),Int'l Units/day +[IU]/10*9{RBCs}, +[IU]/d, +[IU]/dL,Int'l Units/100 mL +[IU]/g, +[IU]/g{Hb}, +[IU]/h,Int'l Units/hr +[IU]/kg, +[IU]/kg/d, +[IU]/L,Int'l Units/L +[IU]/L{37Cel}, +[IU]/mg{creat}, +[IU]/min,Int'l Units/min +[IU]/mL,Int'l Units/mL +[ka'U], +[knk'U], +[lb_av], +[LPF], +[mclg'U], +[mi_i], +[MPL'U], +[MPL'U]/mL, +[oz_av], +[oz_tr], +[pH], +[ppb], +[ppm], +[ppm]{v/v}, +[ppth], +[pptr], +[psi], +[pt_us], +[qt_us], +[sft_i], +[sin_i], +[syd_i], +[tbs_us], +[tb'U], +[todd'U], +[tsp_us], +[yd_i], +{#}, +{#}/[HPF], +{#}/[LPF], +{#}/{platelet}, +{#}/L, +{#}/min, +{#}/mL, +{#}/uL, +{absorbance}, +{activity}, +{AHF'U}, +{APS'U}, +{ARU}, +{beats}/min, +{binding_index}, +{CAE'U}, +{CAG_repeats}, +{cells}, +{cells}/[HPF], +{cells}/uL, +{CH100'U}, +{clock_time}, +{copies}, +{copies}/mL, +{copies}/ug, +{count}, +{CPM}, +{CPM}/10*3{cell}, +{delta_OD}, +{dilution}, +{Ehrlich'U}, +{Ehrlich'U}/(2.h), +{Ehrlich'U}/100.g, +{Ehrlich'U}/d, +{Ehrlich'U}/dL, +{EIA_index}, +{EIA_titer}, +{EIA'U}, +{EIA'U}/U, +{ELISA'U}, +{EV}, +{FIU}, +{fraction}, +{GAA_repeats}, +{genomes}/mL, +{Globules}/[HPF], +{GPS'U}, +{HA_titer}, +{IFA_index}, +{IFA_titer}, +{ImmuneComplex'U}, +{index_val}, +{INR}, +{ISR}, +{JDF'U}, +{JDF'U}/L, +{KCT'U}, +{KRONU'U}/mL, +{Log_copies}/mL, +{Log_IU}, +{Log_IU}/mL, +{Log}, +{Lyme_index_value}, +{M.o.M}, +{minidrop}/min, +{minidrop}/s, +{mm/dd/yyyy}, +{MPS'U}, +{MPS'U}/mL, +{mutation}, +{OD_unit}, +{Pan_Bio'U}, +{percentile}, +{phenotype}, +{ratio}, +{RBC}/uL, +{rel_saturation}, +{Rubella_virus}, +{s_co_ratio}, +{saturation}, +{shift}, +{spermatozoa}/mL, +{STDV}, +{titer}, +{TSI_index}, +{WBCs}, +10*12/L, +10*3, +10*3/L, +10*3/mL, +10*3/uL, +10*3{copies}/mL, +10*3{RBCs}, +10*4/uL, +10*5, +10*6, +10*6.[CFU]/L, +10*6.[IU], +10*6/(24.h), +10*6/kg, +10*6/L, +10*6/mL, +10*6/uL, +10*8, +10*9/L, +10*9/mL, +10*9/uL, +10.L/(min.m2), +10.L/min, +10.uN.s/(cm5.m2), +24.h, +A, +a, +A/m, +ag/{cell}, +atm, +bar,Bar +Bq, +cal, +Cel, +cg, +cL, +cm,cm +cm[H2O], +cm[H2O]/L/s, +cm[H2O]/s/m, +cm[Hg], +cm2,cm2 cust +cm2/s, +cP, +cSt, +d, +daL/min, +daL/min/m2, +dB, +deg, +deg/s, +dg, +dL, +dm, +dm2/s2, +dyn.s/(cm.m2), +dyn.s/cm, +eq, +eq/L, +eq/mL, +eq/mmol, +eq/umol, +erg, +eV, +F, +fg, +fL, +fm, +fmol, +fmol/g, +fmol/L, +fmol/mg, +fmol/mg{cyt_prot}, +fmol/mg{prot}, +fmol/mL, +g,g +g.m, +g.m/{beat}, +g/(100.g), +g/(12.h), +g/(24.h),g/day +g/(3.d), +g/(4.h), +g/(48.h), +g/(5.h), +g/(6.h), +g/(72.h), +g/(8.h){shift}, +g/{specimen}, +g/{total_output}, +g/{total_weight}, +g/cm3, +g/d, +g/dL,g/100 mL +g/g, +g/g{creat}, +g/g{globulin}, +g/g{tissue}, +g/h,g/hr +g/h/m2, +g/kg ,g/kg +g/kg/(8.h), +g/kg/(8.h){shift}, +g/kg/d,g/kg/day +g/kg/h, +g/kg/min, +g/L,g/L +g/m2,g/m2 +g/mg, +g/min, +g/mL,g/mL +g/mmol, +g/mol{creat}, +g{creat}, +g{Hb}, +g{total_nit}, +g{total_prot}, +g{wet_tissue}, +Gy, +H, +h, +Hz, +J, +J/L, +K, +K/W, +k[IU]/L, +k[IU]/mL, +kat, +kat/kg, +kat/L, +kcal,kcal +kcal/[oz_av], +kcal/d,kcal/day +kcal/h, +kcal/kg/(24.h), +kg, +kg.m/s, +kg/(s.m2), +kg/h, +kg/L, +kg/m2, +kg/m3, +kg/min, +kg/mol, +kg/s, +kL, +km, +kPa, +ks, +kU, +kU/g, +kU/L, +kU/L{class}, +kU/mL, +L,L +L/(24.h), +L/(8.h), +L/(min.m2), +L/d,L/day +L/h, +L/kg, +L/L,L/L +L/min,L/min +L/s, +L/s/s2, +lm, +lm.m2, +m,m +m/s, +m/s2, +m[IU]/L, +m[IU]/mL, +m2, +m2/s, +m3/s, +mA, +mbar, +mbar.s/L, +mbar/L/s, +meq,mEq +meq/(2.h), +meq/(24.h), +meq/(8.h), +meq/{specimen}, +meq/{total_volume}, +meq/d,mEq/day +meq/dL,mEq/100 mL +meq/g, +meq/g{creat}, +meq/h,mEq/hr +meq/kg,mEq/kg +meq/kg/h, +meq/L, +meq/m2, +meq/min, +meq/mL, +mg,mg +mg/(10.h), +mg/(12.h), +mg/(2.h), +mg/(24.h), +mg/(6.h), +mg/(72.h), +mg/(8.h), +mg/{collection}, +mg/{specimen}, +mg/{total_output}, +mg/{total_volume}, +mg/d,mg/day +mg/d/{1.73_m2}, +mg/dL, +mg/dL{RBCs}, +mg/g, +mg/g{creat}, +mg/g{dry_tissue}, +mg/g{feces}, +mg/g{tissue}, +mg/g{wet_tissue}, +mg/h,mg/hr +mg/kg,mg/kg +mg/kg/(8.h), +mg/kg/d,mg/kg/day +mg/kg/h,mg/kg/hr +mg/kg/min,mg/kg/min +mg/L,mg/L +mg/L{RBCs}, +mg/m2,mg/m2 +mg/m3, +mg/mg, +mg/mg{creat}, +mg/min,mg/min +mg/mL,mg/mL +mg/mmol, +mg/mmol{creat}, +mg/wk, +mg{FEU}/L, +min, +mL,mL +mL/(10.h), +mL/(12.h), +mL/(2.h), +mL/(24.h), +mL/(4.h), +mL/(5.h), +mL/(6.h), +mL/(72.h), +mL/(8.h), +mL/(8.h)/kg, +mL/[sin_i], +mL/{beat}, +mL/{beat}/m2, +mL/cm[H2O], +mL/d,mL/day +mL/dL, +mL/h,mL/hr +mL/kg,mL/kg +mL/kg/(8.h), +mL/kg/d,mL/kg/day +mL/kg/h,mL/kg/hr +mL/kg/min, +mL/m2, +mL/mbar, +mL/min, +mL/min/{1.73_m2}, +mL/min/m2, +mL/mm, +mL/s, +mL{fetal_RBCs}, +mm,mm +mm/h, +mm/min, +mm[H2O], +mm[Hg], +mm2, +mmol,mmol +mmol/(12.h), +mmol/(2.h), +mmol/(24.h), +mmol/(5.h), +mmol/(6.h), +mmol/(8.h), +mmol/{ejaculate}, +mmol/{specimen}, +mmol/{total_vol}, +mmol/d, +mmol/dL, +mmol/g, +mmol/g{creat}, +mmol/h, +mmol/h/mg{Hb}, +mmol/h/mg{prot}, +mmol/kg,mmol/kg +mmol/kg/(8.h), +mmol/kg/d,mmol/kg/day +mmol/kg/h, +mmol/kg/min, +mmol/L, +mmol/L{RBCs}, +mmol/m2, +mmol/min, +mmol/mmol, +mmol/mmol{creat}, +mmol/mmol{urea}, +mmol/mol, +mmol/mol{creat}, +mmol/s/L, +mo, +mol, +mol/kg, +mol/kg/s, +mol/L, +mol/m3, +mol/mL, +mol/mol, +mol/s, +mosm, +mosm/kg, +mosm/L, +mPa, +mPa.s, +Ms, +ms, +mU/g, +mU/g{Hb}, +mU/g{prot}, +mU/L, +mU/mg, +mU/mg{creat}, +mU/mL, +mU/mL/min, +mU/mmol{creat}, +mU/mmol{RBCs}, +mV, +N, +N.cm, +N.s, +ng,ng +ng/(24.h), +ng/(8.h), +ng/10*6, +ng/10*6{RBCs}, +ng/d, +ng/dL, +ng/g, +ng/g{creat}, +ng/h, +ng/kg,ng/kg +ng/kg/(8.h), +ng/kg/h, +ng/kg/min,ng/kg/min +ng/L, +ng/m2, +ng/mg, +ng/mg/h, +ng/mg{creat}, +ng/mg{prot}, +ng/min, +ng/mL, +ng/mL/h, +ng/mL{RBCs}, +ng/s, +ng/U, +ng{FEU}/mL, +NI, +nkat, +nL, +nm, +nm/s/L, +nmol, +nmol/(24.h), +nmol/d, +nmol/dL, +nmol/dL{GF}, +nmol/g, +nmol/g{creat}, +nmol/g{dry_wt}, +nmol/h/L, +nmol/h/mg{prot}, +nmol/L, +nmol/L/mmol{creat}, +nmol/L{RBCs}, +nmol/m/mg{prot}, +nmol/mg, +nmol/mg/h, +nmol/mg{creat}, +nmol/mg{prot}, +nmol/mg{prot}, +nmol/mg{prot}/h, +nmol/min, +nmol/min/10*6{cells}, +nmol/min/mg{Hb}, +nmol/min/mg{prot}, +nmol/min/mL, +nmol/mL, +nmol/mL/h, +nmol/mL/min, +nmol/mmol, +nmol/mmol{creat}, +nmol/mmol{creat}, +nmol/mol, +nmol/nmol, +nmol/s, +nmol/s/L, +nmol/umol{creat}, +nmol{ATP}, +nmol{BCE}, +nmol{BCE}/L, +ns, +nU/{RBC}, +nU/mL, +Ohm, +Ohm.m, +osm, +osm/kg, +osm/L, +OT, +Pa, +pA, +pg, +pg/{cell}, +pg/{RBC}, +pg/dL, +pg/L, +pg/mg, +pg/mg{creat}, +pg/mL, +pg/mm, +pkat, +pL, +pm, +pmol, +pmol/(24.h), +pmol/{RBC}, +pmol/d, +pmol/dL, +pmol/g, +pmol/h/mg{prot}, +pmol/h/mL, +pmol/L, +pmol/mg{prot}, +pmol/min, +pmol/min/mg{prot}, +pmol/mL, +pmol/mmol{creat}, +pmol/umol, +pmol/umol{creat}, +ps, +pT, +s, +S, +s/{control}, +Sv, +t, +T, +Torr, +U,Units +U/(10.g){feces}, +U/(12.h), +U/(2.h), +U/(24.h), +U/10, +U/10*10, +U/10*10{cells}, +U/10*12, +U/10*12{RBCs}, +U/10*6, +U/10*9, +U/d,Units/day +U/dL,Units/100 mL +U/g, +U/g{creat}, +U/g{Hb}, +U/g{protein}, +U/h,Units/hr +U/kg{Hb}, +U/L,Units/L +U/min,Units/min +U/mL,Units/mL +U/mL{RBCs}, +U/mmol{creat}, +U/s, +u[IU], +u[IU]/L, +u[IU]/mL, +U{25Cel}/L, +U{37Cel}/L, +ueq, +ueq/L, +ueq/mL, +ug,mcg +ug/(100.g), +ug/(24.h),mcg/day +ug/(8.h), +ug/[sft_i], +ug/{specimen}, +ug/d, +ug/dL, +ug/dL{RBCs}, +ug/g, +ug/g{creat}, +ug/g{dry_tissue}, +ug/g{dry_wt}, +ug/g{feces}, +ug/g{hair}, +ug/g{Hb}, +ug/g{tissue}, +ug/h,mcg/hr +ug/kg,mcg/kg +ug/kg/(8.h), +ug/kg/d,mcg/kg/day +ug/kg/h,mcg/kg/hr +ug/kg/min,mcg/kg/min +ug/L,mcg/L +ug/L/(24.h), +ug/L{RBCs}, +ug/m2,mcg/m2 +ug/m3, +ug/mg, +ug/mg{creat}, +ug/min, +ug/mL,mcg/mL +ug/mL{class}, +ug/mL{eqv}, +ug/mmol, +ug/mmol{creat}, +ug/ng, +ug{FEU}/mL, +ukat, +uL,mcL +uL/(2.h), +uL/h, +um, +um/s, +umol, +umol/(2.h), +umol/(24.h), +umol/(8.h), +umol/d, +umol/dL, +umol/dL{GF}, +umol/g, +umol/g{creat}, +umol/g{Hb}, +umol/h, +umol/kg, +umol/kg{feces}, +umol/L, +umol/L/h, +umol/L{RBCs}, +umol/mg, +umol/mg{creat}, +umol/min, +umol/min/g, +umol/min/g{mucosa}, +umol/min/g{prot}, +umol/min/L, +umol/mL, +umol/mL/min, +umol/mmol, +umol/mmol{creat}, +umol/mol, +umol/mol{creat}, +umol/mol{Hb}, +umol/umol, +umol/umol{creat}, +umol{BCE}/mol, +UN, +uOhm, +us, +uU/g, +uU/L, +uU/mL, +uV, +V, +Wb, +wk, diff --git a/i2p_tasks.py b/i2p_tasks.py index 3cc140c..70745f2 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -320,6 +320,16 @@ def requires(self) -> List[luigi.Task]: return [pcornet_init()] +class loadUnitMap(LoadCSV): + taskName = 'UNIT_MAP' + # unit_map.csv matches values in the CDM spec's _unit spreadsheet + # to unit values from Epic's zc_med_unit table. + csvname = 'curated_data/unit_map.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + class NPIDownloadConfig(luigi.Config): # The configured 'path' and 'npi' variables are used by the downloadNPI method to fetch and # store the NPPES zip file. Changes to these may require changes to the file system. From c35e4c6af7c8d66fce01e66d2fc179dc5f1dd2bc Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 17 Jul 2018 17:52:23 -0500 Subject: [PATCH 420/507] Fixing alter table after changing provider_id column alias. --- Oracle/prescribing.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index cdf8471..519823c 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -94,7 +94,7 @@ join unit_map um on rx.units_cd = um.unit_name where rx.modifier_cd in ('MedObs:Inpatient', 'MedObs:Outpatient') / -alter table prescribing_key modify (provider_id null) +alter table prescribing_key modify (rx_providerid null) / /** prescribing_w_cui From 6a94473ec6d25e9f861a622c85bd77ddb51d21e5 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 18 Jul 2018 09:19:35 -0500 Subject: [PATCH 421/507] Add drop for new route table, remove drop for elminated dose table. --- Oracle/prescribing.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 519823c..b18269c 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -35,11 +35,11 @@ PMN_DROPSQL('drop table prescribing_w_basis'); END; / BEGIN -PMN_DROPSQL('drop table prescribing_w_dose'); +PMN_DROPSQL('drop table prescribing_w_prn'); END; / BEGIN -PMN_DROPSQL('drop table prescribing_w_prn'); +PMN_DROPSQL('drop table prescribing_w_route'); END; / BEGIN From 62dfde7244997a7f660582259832624b5dbd80b5 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 18 Jul 2018 09:31:59 -0500 Subject: [PATCH 422/507] Correcting flubbed identification of dose columns. --- Oracle/prescribing.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index b18269c..bfd1ff4 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -85,7 +85,7 @@ select cast(prescribing_seq.nextval as varchar(19)) prescribingid , concept_cd , modifier_cd , case when trim(translate(tval_char, '0123456789.', ' ')) is null then tval_char else null end rx_dose_ordered -, case when trim(translate(tval_char, '0123456789.', ' ')) is null then um.code else null end rx_dose_order +, case when trim(translate(tval_char, '0123456789.', ' ')) is null then um.code else null end rx_dose_order_units , tval_char raw_rx_dose_ordered , units_cd raw_rx_dose_ordered_unit from blueherondata.observation_fact rx @@ -261,8 +261,8 @@ select rx.prescribingid , rx.raw_rxnorm_cui , cast(null as varchar(50)) raw_rx_quantity , cast(null as varchar(50)) raw_rx_ndc -, rx.rx_dose_ordered raw_rx_dose_ordered -, rx.rx_dose_ordered_unit raw_rx_dose_ordered_unit +, rx.raw_rx_dose_ordered +, rx.raw_rx_dose_ordered_unit , rx.raw_rx_route , cast(null as varchar(50)) raw_rx_refills From 90f74f88e2470631240dbc4d76071e4db2db0cb1 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 18 Jul 2018 09:34:29 -0500 Subject: [PATCH 423/507] Restoring luigi return code values to configuration. These were unintentionally deleted during config cleanup. --- client.cfg | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/client.cfg b/client.cfg index 580224b..f276b85 100644 --- a/client.cfg +++ b/client.cfg @@ -28,6 +28,17 @@ ssh_tunnel= encounter_mapping=1 patient_mapping=1 +[retcode] +# see also http://luigi.readthedocs.io/en/stable/configuration.html +# The following return codes are the recommended exit codes for Luigi +# They are in increasing level of severity (for most applications) +already_running=10 +missing_data=20 +not_run=25 +task_failed=30 +scheduling_error=35 +unhandled_exception=40 + [I2PConfig] datamart_id = C4UK datamart_name = University of Kansas From e088d75fbdd97c3013107e45c473edb56dcecb5b Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 18 Jul 2018 09:38:46 -0500 Subject: [PATCH 424/507] Another typo. Correct name for rx_dose_ordered_units column. --- Oracle/prescribing.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index bfd1ff4..eae02c0 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -85,7 +85,7 @@ select cast(prescribing_seq.nextval as varchar(19)) prescribingid , concept_cd , modifier_cd , case when trim(translate(tval_char, '0123456789.', ' ')) is null then tval_char else null end rx_dose_ordered -, case when trim(translate(tval_char, '0123456789.', ' ')) is null then um.code else null end rx_dose_order_units +, case when trim(translate(tval_char, '0123456789.', ' ')) is null then um.code else null end rx_dose_ordered_units , tval_char raw_rx_dose_ordered , units_cd raw_rx_dose_ordered_unit from blueherondata.observation_fact rx From 3979885d7dacac91b2cb46f785d2f2e4c3382a28 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 18 Jul 2018 09:42:47 -0500 Subject: [PATCH 425/507] Correcting name of rx_dose_ordered_unit column. Don't look too closely at the commit history. This pile of typo corrections is getting embarrassing. --- Oracle/prescribing.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index eae02c0..8eb6640 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -85,7 +85,7 @@ select cast(prescribing_seq.nextval as varchar(19)) prescribingid , concept_cd , modifier_cd , case when trim(translate(tval_char, '0123456789.', ' ')) is null then tval_char else null end rx_dose_ordered -, case when trim(translate(tval_char, '0123456789.', ' ')) is null then um.code else null end rx_dose_ordered_units +, case when trim(translate(tval_char, '0123456789.', ' ')) is null then um.code else null end rx_dose_ordered_unit , tval_char raw_rx_dose_ordered , units_cd raw_rx_dose_ordered_unit from blueherondata.observation_fact rx From c26e758d235cd50b0a4be392138d87d57b1e48f5 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 18 Jul 2018 17:10:56 -0500 Subject: [PATCH 426/507] Untested data_step for 4.1, med_admin minus days supply, unit_map tweaks. --- SAS/data_step_view_inspect.sas | 24 +++++++++ SAS/data_step_view_prep.sas | 91 +++++++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/SAS/data_step_view_inspect.sas b/SAS/data_step_view_inspect.sas index 6c42b27..c558e67 100644 --- a/SAS/data_step_view_inspect.sas +++ b/SAS/data_step_view_inspect.sas @@ -106,6 +106,30 @@ proc print data=sasdata.DEATH_CAUSE (firstobs=1 obs=10); run; +proc contents data=sasdata.PROVIDER; +run; +proc print data=sasdata.PROVIDER (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.OBS_CLIN; +run; +proc print data=sasdata.OBS_CLIN (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.OBS_GEN; +run; +proc print data=sasdata.OBS_GEN (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.MED_ADMIN; +run; +proc print data=sasdata.MED_ADMIN (firstobs=1 obs=10); +run; + + proc contents data=sasdata.HARVEST; run; proc print data=sasdata.HARVEST (firstobs=1 obs=10); diff --git a/SAS/data_step_view_prep.sas b/SAS/data_step_view_prep.sas index d055c9f..33cde2d 100644 --- a/SAS/data_step_view_prep.sas +++ b/SAS/data_step_view_prep.sas @@ -12,7 +12,7 @@ ODS HTML; ***************************************************************; -* Include configurable SAS libraies +* Include configurable SAS libraries ***************************************************************; %let fpath=%sysget(SAS_EXECFILEPATH); %let fname=%sysget(SAS_EXECFILENAME); @@ -270,6 +270,82 @@ data sasdata.DEATH_CAUSE / view=sasdata.DEATH_CAUSE; run; +***************************************************************; +* Create data step view for PROVIDER +***************************************************************; +data sasdata.PROVIDER / view=sasdata.PROVIDER; + set oracdata.PROVIDER; +run; + + +***************************************************************; +* Create data step view for OBS_CLIN +***************************************************************; +data sasdata.OBS_CLIN / view=sasdata.OBS_CLIN; + set oracdata.OBS_CLIN( + rename = ( + OBSCLIN_TIME = _OBSCLIN_TIME + ) + ) + ; + + OBSCLIN_DATE = datepart(OBSCLIN_DATE); + format OBSCLIN_DATE mmddyy10.; + + OBSCLIN_TIME = input(_OBSCLIN_TIME, hhmmss.); + format OBSCLIN_TIME hhmm.; + drop _OBSCLIN_TIME; +run; + + +***************************************************************; +* Create data step view for OBS_GEN +***************************************************************; +data sasdata.OBS_GEN / view=sasdata.OBS_GEN; + set oracdata.OBS_GEN( + rename = ( + OBSGEN_TIME = _OBSGEN_TIME + ) + ) + ; + + OBSGEN_DATE = datepart(OBSGEN_DATE); + format OBSGEN_DATE mmddyy10.; + + OBSGEN_TIME = input(_OBSGEN_TIME, hhmmss.); + format OBSGEN_TIME hhmm.; + drop _OBSGEN_TIME; +run; + + +***************************************************************; +* Create data step view for MED_ADMIN +***************************************************************; +data sasdata.MED_ADMIN / view=sasdata.MED_ADMIN; + set oracdata.MED_ADMIN( + rename = ( + MEDADMIN_START_TIME = _MEDADMIN_START_TIME + MEDADMIN_STOP_TIME = _MEDADMIN_STOP_TIME + ) + ) + ; + + MEDADMIN_START_DATE = datepart(MEDADMIN_START_DATE); + format MEDADMIN_START_DATE mmddyy10.; + + MEDADMIN_START_TIME = input(_MEDADMIN_START_TIME, hhmmss.); + format MEDADMIN_START_TIME hhmm.; + drop _MEDADMIN_START_TIME; + + MEDADMIN_STOP_DATE = datepart(MEDADMIN_STOP_DATE); + format MEDADMIN_STOP_DATE mmddyy10.; + + MEDADMIN_STOP_TIME = input(_MEDADMIN_STOP_TIME, hhmmss.); + format MEDADMIN_STOP_TIME hhmm.; + drop _MEDADMIN_STOP_TIME; +run; + + ***************************************************************; * Create data step view for HARVEST ***************************************************************; @@ -317,4 +393,17 @@ data sasdata.HARVEST / view=sasdata.HARVEST; REFRESH_DEATH_CAUSE_DATE = datepart(REFRESH_DEATH_CAUSE_DATE); format REFRESH_DEATH_CAUSE_DATE mmddyy10.; + + REFRESH_MED_ADMIN_DATE = datepart(REFRESH_MED_ADMIN_DATE); + format REFRESH_MED_ADMIN_DATE mmddyy10.; + + REFRESH_OBS_CLIN_DATE = datepart(REFRESH_OBS_CLIN_DATE); + format REFRESH_OBS_CLIN_DATE mmddyy10.; + + REFRESH_PROVIDER_DATE = datepart(REFRESH_PROVIDER_DATE); + format REFRESH_PROVIDER_DATE mmddyy10.; + + REFRESH_OBS_GEN_DATE = datepart(REFRESH_OBS_GEN_DATE); + format REFRESH_OBS_GEN_DATE mmddyy10.; + run; From bf6af2c4a663c8202e2d6d8b74741a8e4f0f22a9 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 18 Jul 2018 17:12:51 -0500 Subject: [PATCH 427/507] med_admin minus days supply modifiers and unit_map tweak Missed these files in the previous commit with similar message. --- Oracle/med_admin.sql | 62 ++++++++++++++++++++++++++++----------- curated_data/unit_map.csv | 2 ++ 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 293b38d..cc85520 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -49,10 +49,23 @@ PMN_DROPSQL('drop index med_admin_idx'); execute immediate 'truncate table med_admin'; -insert into med_admin(patid, encounterid, medadmin_providerid, medadmin_start_date, medadmin_start_time, medadmin_stop_date, -medadmin_stop_time, medadmin_type, medadmin_code, medadmin_dose_admin, medadmin_dose_admin_unit, medadmin_source, raw_medadmin_med_name, -raw_medadmin_code, raw_medadmin_dose_admin, raw_medadmin_dose_admin_unit, raw_medadmin_route) - +insert into med_admin(patid + , encounterid + , medadmin_providerid + , medadmin_start_date + , medadmin_start_time + , medadmin_stop_date + , medadmin_stop_time + , medadmin_type + , medadmin_code + , medadmin_dose_admin + , medadmin_dose_admin_unit + , medadmin_source + , raw_medadmin_med_name + , raw_medadmin_code + , raw_medadmin_dose_admin + , raw_medadmin_dose_admin_unit + , raw_medadmin_route) with med_start as ( select patient_num, encounter_num, provider_id, start_date, end_date, concept_cd, modifier_cd, instance_num from BLUEHERONDATA.observation_fact @@ -78,9 +91,23 @@ with med_start as ( or modifier_cd = 'MedObs|MAR:Bolus from Syringe' or modifier_cd = 'MedObs|MAR:Bolus.' ) -select med_start.patient_num, med_start.encounter_num, med_start.provider_id, med_start.start_date, to_char(med_start.start_date, 'HH24:MI'), med_start.end_date, -to_char(med_start.end_date, 'HH24:MI'), 'RX', med_p.pcori_basecode, med_dose.nval_num, med_dose.units_cd, 'OD', med_p.c_name, med_start.concept_cd, med_dose.nval_num, -med_dose.units_cd, med_start.modifier_cd +select med_start.patient_num + , med_start.encounter_num + , med_start.provider_id + , med_start.start_date + , to_char(med_start.start_date, 'HH24:MI') + , med_start.end_date + , to_char(med_start.end_date, 'HH24:MI') + , 'RX' + , med_p.pcori_basecode + , med_dose.nval_num + , case when nval_num is null then null else nvl(um.code, 'OT') end + , 'OD' + , med_p.c_name + , med_start.concept_cd + , med_dose.nval_num + , med_dose.units_cd + , med_start.modifier_cd from med_start left join BLUEHERONDATA.observation_fact med_dose on med_dose.instance_num = med_start.instance_num @@ -93,16 +120,17 @@ or med_dose.modifier_cd = 'MedObs:MAR_Dose|tab' or med_dose.modifier_cd = 'MedObs:MAR_Dose|units' or med_dose.modifier_cd = 'MedObs:MAR_Dose|l' or med_dose.modifier_cd = 'MedObs:MAR_Dose|mg' -or med_dose.modifier_cd = 'MedObs:Dose|puff' -or med_dose.modifier_cd = 'MedObs:Dose|drop' -or med_dose.modifier_cd = 'MedObs:Dose|cap' -or med_dose.modifier_cd = 'MedObs:Dose|meq' -or med_dose.modifier_cd = 'MedObs:Dose|units' -or med_dose.modifier_cd = 'MedObs:Dose|l' -or med_dose.modifier_cd = 'MedObs:Dose|tab' -or med_dose.modifier_cd = 'MedObs:Dose|mg') -left join BLUEHERONMETADATA.pcornet_med med_p -on med_p.c_basecode = med_start.concept_cd +--or med_dose.modifier_cd = 'MedObs:Dose|puff' +--or med_dose.modifier_cd = 'MedObs:Dose|drop' +--or med_dose.modifier_cd = 'MedObs:Dose|cap' +--or med_dose.modifier_cd = 'MedObs:Dose|meq' +--or med_dose.modifier_cd = 'MedObs:Dose|units' +--or med_dose.modifier_cd = 'MedObs:Dose|l' +--or med_dose.modifier_cd = 'MedObs:Dose|tab' +--or med_dose.modifier_cd = 'MedObs:Dose|mg' +) +left join BLUEHERONMETADATA.pcornet_med med_p on med_p.c_basecode = med_start.concept_cd +left join unit_map um on um.unit_name = med_dose.units_cd ; execute immediate 'create index med_admin_idx on med_admin (PATID, ENCOUNTERID)'; diff --git a/curated_data/unit_map.csv b/curated_data/unit_map.csv index 5b9d3f2..f69edb3 100644 --- a/curated_data/unit_map.csv +++ b/curated_data/unit_map.csv @@ -393,6 +393,7 @@ kU/L, kU/L{class}, kU/mL, L,L +L,l L/(24.h), L/(8.h), L/(min.m2), @@ -476,6 +477,7 @@ mg/wk, mg{FEU}/L, min, mL,mL +mL,ml mL/(10.h), mL/(12.h), mL/(2.h), From a415b076a2f610ffcf26fbbbf17799fa0b03fc11 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 18 Jul 2018 17:49:21 -0500 Subject: [PATCH 428/507] Making downloadNPI dependent on pcornet_int, plus comments. Without the dependency, downloadNPI was racing pcornet_int and failing. --- i2p_tasks.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index 70745f2..c0cee58 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -203,7 +203,13 @@ def results(self) -> List[RowProxy]: except DatabaseError: return [] - +# TODO: pcornet_init drops and recreates the cdm_status table, forcing all tasks to wait until init is done. +# Moving this operation to a distinct task would allow some other tasks (e.g. mapping tasks) to proceed, while init +# performs other labor intesive SQL operations. +# In the mean time, don't forget to make all tasks that use the status table dependent on pcornet_init. +# TODO: On a related matter, if the cdm_status table is missing (e.g. on a db where CDM has never been run), running +# the full pipeline, starting at pcornet_loader, will fail. It would be nice to detect this situation and first run +# pcornet_init before attempting other tasks. class pcornet_init(I2PScriptTask): script = Script.pcornet_init @@ -366,6 +372,9 @@ class loadSpecialtyCode(LoadCSV): # It maps a provider specialty code to a descriptive text and grouping. csvname = 'curated_data/provider_specialty_code.csv' + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + class downloadNPI(CDMStatusTask): ''' @@ -393,6 +402,9 @@ def fetch(self) -> None: def unzip(self) -> None: subprocess.call(['unzip', '-o', self.dl_path + self.npi_zip, '-d', self.dl_path]) # ISSUE: ambient + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + class extractNPI(CDMStatusTask): ''' From 984b8ed2c2100383ab59f2c0fbe6d1e06c4a2362 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 19 Jul 2018 10:31:42 -0500 Subject: [PATCH 429/507] Adding map load tasks to dependent tables, plus minor fixes. Missed the dependencies in piecewise testing as the map tables were always available. Found the bug when moving to a clean environment, as stored procs wouldn't compile. --- Oracle/med_admin.sql | 6 +- SAS/data_step_view_inspect_3_1.sas | 112 ++++++++++ SAS/data_step_view_prep_3_1.sas | 320 +++++++++++++++++++++++++++++ client.cfg | 2 +- i2p_tasks.py | 16 +- 5 files changed, 446 insertions(+), 10 deletions(-) create mode 100644 SAS/data_step_view_inspect_3_1.sas create mode 100644 SAS/data_step_view_prep_3_1.sas diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index cc85520..cba71a8 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -51,6 +51,7 @@ execute immediate 'truncate table med_admin'; insert into med_admin(patid , encounterid + , prescribingid , medadmin_providerid , medadmin_start_date , medadmin_start_time @@ -60,6 +61,7 @@ insert into med_admin(patid , medadmin_code , medadmin_dose_admin , medadmin_dose_admin_unit + , medadmin_route , medadmin_source , raw_medadmin_med_name , raw_medadmin_code @@ -93,6 +95,7 @@ with med_start as ( ) select med_start.patient_num , med_start.encounter_num + , null , med_start.provider_id , med_start.start_date , to_char(med_start.start_date, 'HH24:MI') @@ -102,12 +105,13 @@ select med_start.patient_num , med_p.pcori_basecode , med_dose.nval_num , case when nval_num is null then null else nvl(um.code, 'OT') end + , null , 'OD' , med_p.c_name , med_start.concept_cd , med_dose.nval_num , med_dose.units_cd - , med_start.modifier_cd + , med_start.modifier_cd -- Modifier rather than raw route. from med_start left join BLUEHERONDATA.observation_fact med_dose on med_dose.instance_num = med_start.instance_num diff --git a/SAS/data_step_view_inspect_3_1.sas b/SAS/data_step_view_inspect_3_1.sas new file mode 100644 index 0000000..6c42b27 --- /dev/null +++ b/SAS/data_step_view_inspect_3_1.sas @@ -0,0 +1,112 @@ +/******************************************************************************* +* Inspect the PCORnet CDMv3 data step views created by data_step_view_prep.sas. +* Runs the SAS content procedure over and gets the first ten records from each +* of the data step views. +*******************************************************************************/ + + +***************************************************************; +* Clear SAS result buffer +***************************************************************; +ODS HTML CLOSE; +ODS HTML; + + +***************************************************************; +* Include configurable SAS libraies +***************************************************************; +%let fpath=%sysget(SAS_EXECFILEPATH); +%let fname=%sysget(SAS_EXECFILENAME); +%let path= %sysfunc(tranwrd(&fpath,&fname,'')); +%put &path; +%include '&path/configuration.sas'; + + +proc contents data=sasdata.DEMOGRAPHIC; +run; +proc print data=sasdata.DEMOGRAPHIC (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.ENROLLMENT; +run; +proc print data=sasdata.ENROLLMENT (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.ENCOUNTER; +run; +proc print data=sasdata.ENCOUNTER (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.DIAGNOSIS; +run; +proc print data=sasdata.DIAGNOSIS (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.PROCEDURES; +run; +proc print data=sasdata.PROCEDURES (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.VITAL; +run; +proc print data=sasdata.VITAL (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.DISPENSING; +run; +proc print data=sasdata.DISPENSING (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.LAB_RESULT_CM; +run; +proc print data=sasdata.LAB_RESULT_CM (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.CONDITION; +run; +proc print data=sasdata.CONDITION (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.PRO_CM; +run; +proc print data=sasdata.PRO_CM (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.PRESCRIBING; +run; +proc print data=sasdata.PRESCRIBING (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.PCORNET_TRIAL; +run; +proc print data=sasdata.PCORNET_TRIAL (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.DEATH; +run; +proc print data=sasdata.DEATH (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.DEATH_CAUSE; +run; +proc print data=sasdata.DEATH_CAUSE (firstobs=1 obs=10); +run; + + +proc contents data=sasdata.HARVEST; +run; +proc print data=sasdata.HARVEST (firstobs=1 obs=10); +run; diff --git a/SAS/data_step_view_prep_3_1.sas b/SAS/data_step_view_prep_3_1.sas new file mode 100644 index 0000000..e54d72d --- /dev/null +++ b/SAS/data_step_view_prep_3_1.sas @@ -0,0 +1,320 @@ +/******************************************************************************* +* Generate the PCORnet CDMv3 data step views required by PCORnet SAS queries, +* providing the required data type transformations where needed. +*******************************************************************************/ + + +***************************************************************; +* Clear SAS result buffer +***************************************************************; +ODS HTML CLOSE; +ODS HTML; + + +***************************************************************; +* Include configurable SAS libraies +***************************************************************; +%let fpath=%sysget(SAS_EXECFILEPATH); +%let fname=%sysget(SAS_EXECFILENAME); +%let path= %sysfunc(tranwrd(&fpath,&fname,'')); +%put &path; +%include '&path/configuration.sas'; + + +***************************************************************; +* Create data step view for DEMOGRAPHIC +***************************************************************; +data sasdata.DEMOGRAPHIC / view=sasdata.DEMOGRAPHIC; + set oracdata.DEMOGRAPHIC( + rename = ( + BIRTH_TIME = _BIRTH_TIME + ) + ) + ; + + BIRTH_DATE = datepart(BIRTH_DATE); + format BIRTH_DATE mmddyy10.; + + BIRTH_TIME = input(_BIRTH_TIME, hhmmss.); + format BIRTH_TIME hhmm.; + drop _BIRTH_TIME; +run; + + +***************************************************************; +* Create data step view for ENROLLMENT +***************************************************************; +data sasdata.ENROLLMENT / view=sasdata.ENROLLMENT; + set oracdata.ENROLLMENT; + + ENR_START_DATE = datepart(ENR_START_DATE); + format ENR_START_DATE mmddyy10.; + + ENR_END_DATE = datepart(ENR_END_DATE); + format ENR_END_DATE mmddyy10.; + +run; + + +***************************************************************; +* Create data step view for ENCOUNTER +***************************************************************; +data sasdata.ENCOUNTER / view=sasdata.ENCOUNTER; + set oracdata.ENCOUNTER( + rename = ( + ADMIT_TIME = _ADMIT_TIME + DISCHARGE_TIME = _DISCHARGE_TIME + ) + ) + ; + + ADMIT_DATE = datepart(ADMIT_DATE); + format ADMIT_DATE mmddyy10.; + + ADMIT_TIME = input(_ADMIT_TIME, hhmmss.); + format ADMIT_TIME hhmm.; + drop _ADMIT_TIME; + + DISCHARGE_DATE = datepart(DISCHARGE_DATE); + format DISCHARGE_DATE mmddyy10.; + + DISCHARGE_TIME = input(_DISCHARGE_TIME, hhmmss.); + format DISCHARGE_TIME hhmm.; + drop _DISCHARGE_TIME; +run; + + +***************************************************************; +* Create data step view for DIAGNOSIS +***************************************************************; +data sasdata.DIAGNOSIS / view=sasdata.DIAGNOSIS; + set oracdata.DIAGNOSIS; + + ADMIT_DATE = datepart(ADMIT_DATE); + format ADMIT_DATE mmddyy10.; +run; + + +***************************************************************; +* Create data step view for PROCEDURES +***************************************************************; +data sasdata.PROCEDURES / view=sasdata.PROCEDURES; + set oracdata.PROCEDURES; + + ADMIT_DATE = datepart(ADMIT_DATE); + format ADMIT_DATE mmddyy10.; + + PX_DATE = datepart(PX_DATE); + format PX_DATE mmddyy10.; +run; + + +***************************************************************; +* Create data step view for VITAL +***************************************************************; +data sasdata.VITAL / view=sasdata.VITAL; + set oracdata.VITAL( + rename = ( + MEASURE_TIME = _MEASURE_TIME + ) + ) + ; + + MEASURE_DATE = datepart(MEASURE_DATE); + format MEASURE_DATE mmddyy10.; + + MEASURE_TIME = input(_MEASURE_TIME, hhmmss.); + format MEASURE_TIME hhmm.; + drop _MEASURE_TIME; +run; + + +***************************************************************; +* Create data step view for DISPENSING +***************************************************************; +data sasdata.DISPENSING / view=sasdata.DISPENSING; + set oracdata.DISPENSING; + + DISPENSE_DATE = datepart(DISPENSE_DATE); + format DISPENSE_DATE mmddyy10.; +run; + + +***************************************************************; +* Create data step view for LAB_RESULT_CM +***************************************************************; +data sasdata.LAB_RESULT_CM / view=sasdata.LAB_RESULT_CM; + set oracdata.LAB_RESULT_CM( + rename = ( + RESULT_TIME = _RESULT_TIME + SPECIMEN_TIME = _SPECIMEN_TIME + ) + ) + ; + + LAB_ORDER_DATE = datepart(LAB_ORDER_DATE); + format LAB_ORDER_DATE mmddyy10.; + + RESULT_DATE = datepart(RESULT_DATE); + format RESULT_DATE mmddyy10.; + + RESULT_TIME = input(_RESULT_TIME, hhmmss.); + format RESULT_TIME hhmm.; + drop _RESULT_TIME; + + SPECIMEN_DATE = datepart(SPECIMEN_DATE); + format SPECIMEN_DATE mmddyy10.; + + SPECIMEN_TIME = input(_SPECIMEN_TIME, hhmmss.); + format SPECIMEN_TIME hhmm.; + drop _SPECIMEN_TIME; +run; + + +***************************************************************; +* Create data step view for CONDITION +***************************************************************; +data sasdata.CONDITION / view=sasdata.CONDITION; + set oracdata.CONDITION; + + REPORT_DATE = datepart(REPORT_DATE); + format REPORT_DATE mmddyy10.; + + RESOLVE_DATE = datepart(RESOLVE_DATE); + format RESOLVE_DATE mmddyy10.; + + ONSET_DATE = datepart(ONSET_DATE); + format ONSET_DATE mmddyy10.; +run; + + +***************************************************************; +* Create data step view for PRO_CM +***************************************************************; +data sasdata.PRO_CM / view=sasdata.PRO_CM; + set oracdata.PRO_CM( + rename = ( + PRO_TIME = _PRO_TIME + ) + ) + ; + + PRO_DATE = datepart(PRO_DATE); + format PRO_DATE mmddyy10.; + + PRO_TIME = input(_PRO_TIME, hhmmss.); + format PRO_TIME hhmm.; + drop _PRO_TIME; +run; + + +***************************************************************; +* Create data step view for PRESCRIBING +***************************************************************; +data sasdata.PRESCRIBING / view=sasdata.PRESCRIBING; + set oracdata.PRESCRIBING( + rename = ( + RX_ORDER_TIME = _RX_ORDER_TIME + ) + ) + ; + + RX_ORDER_DATE = datepart(RX_ORDER_DATE); + format RX_ORDER_DATE mmddyy10.; + + RX_ORDER_TIME = input(_RX_ORDER_TIME, hhmmss.); + format RX_ORDER_TIME hhmm.; + drop _RX_ORDER_TIME; + + RX_START_DATE = datepart(RX_START_DATE); + format RX_START_DATE mmddyy10.; + + RX_END_DATE = datepart(RX_END_DATE); + format RX_END_DATE mmddyy10.; +run; + + +***************************************************************; +* Create data step view for PCORNET_TRIAL +***************************************************************; +data sasdata.PCORNET_TRIAL / view=sasdata.PCORNET_TRIAL; + set oracdata.PCORNET_TRIAL; + + TRIAL_ENROLL_DATE = datepart(TRIAL_ENROLL_DATE); + format TRIAL_ENROLL_DATE mmddyy10.; + + TRIAL_END_DATE = datepart(TRIAL_END_DATE); + format TRIAL_END_DATE mmddyy10.; + + TRIAL_WITHDRAW_DATE = datepart(TRIAL_WITHDRAW_DATE); + format TRIAL_WITHDRAW_DATE mmddyy10.; +run; + + +***************************************************************; +* Create data step view for DEATH +***************************************************************; +data sasdata.DEATH / view=sasdata.DEATH; + set oracdata.DEATH; + + DEATH_DATE = datepart(DEATH_DATE); + format DEATH_DATE mmddyy10.; +run; + + +***************************************************************; +* Create data step view for DEATH_CAUSE +***************************************************************; +data sasdata.DEATH_CAUSE / view=sasdata.DEATH_CAUSE; + set oracdata.DEATH_CAUSE; +run; + + +***************************************************************; +* Create data step view for HARVEST +***************************************************************; +data sasdata.HARVEST / view=sasdata.HARVEST; + set oracdata.HARVEST; + + REFRESH_DEMOGRAPHIC_DATE = datepart(REFRESH_DEMOGRAPHIC_DATE); + format REFRESH_DEMOGRAPHIC_DATE mmddyy10.; + + REFRESH_ENROLLMENT_DATE = datepart(REFRESH_ENROLLMENT_DATE); + format REFRESH_ENROLLMENT_DATE mmddyy10.; + + REFRESH_ENCOUNTER_DATE = datepart(REFRESH_ENCOUNTER_DATE); + format REFRESH_ENCOUNTER_DATE mmddyy10.; + + REFRESH_DIAGNOSIS_DATE = datepart(REFRESH_DIAGNOSIS_DATE); + format REFRESH_DIAGNOSIS_DATE mmddyy10.; + + REFRESH_PROCEDURES_DATE = datepart(REFRESH_PROCEDURES_DATE); + format REFRESH_PROCEDURES_DATE mmddyy10.; + + REFRESH_VITAL_DATE = datepart(REFRESH_VITAL_DATE); + format REFRESH_VITAL_DATE mmddyy10.; + + REFRESH_DISPENSING_DATE = datepart(REFRESH_DISPENSING_DATE); + format REFRESH_DISPENSING_DATE mmddyy10.; + + REFRESH_LAB_RESULT_CM_DATE = datepart(REFRESH_LAB_RESULT_CM_DATE); + format REFRESH_LAB_RESULT_CM_DATE mmddyy10.; + + REFRESH_CONDITION_DATE = datepart(REFRESH_CONDITION_DATE); + format REFRESH_CONDITION_DATE mmddyy10.; + + REFRESH_PRO_CM_DATE = datepart(REFRESH_PRO_CM_DATE); + format REFRESH_PRO_CM_DATE mmddyy10.; + + REFRESH_PRESCRIBING_DATE = datepart(REFRESH_PRESCRIBING_DATE); + format REFRESH_PRESCRIBING_DATE mmddyy10.; + + REFRESH_PCORNET_TRIAL_DATE = datepart(REFRESH_PCORNET_TRIAL_DATE); + format REFRESH_PCORNET_TRIAL_DATE mmddyy10.; + + REFRESH_DEATH_DATE = datepart(REFRESH_DEATH_DATE); + format REFRESH_DEATH_DATE mmddyy10.; + + REFRESH_DEATH_CAUSE_DATE = datepart(REFRESH_DEATH_CAUSE_DATE); + format REFRESH_DEATH_CAUSE_DATE mmddyy10.; +run; diff --git a/client.cfg b/client.cfg index f276b85..d47a982 100644 --- a/client.cfg +++ b/client.cfg @@ -44,7 +44,7 @@ datamart_id = C4UK datamart_name = University of Kansas enrollment_months_back = 42 i2b2_data_schema = BLUEHERONDATA -i2b2_etl_schema = HERON_ETL_3 +i2b2_etl_schema = HERON_ETL_1 i2b2_meta_schema = BLUEHERONMETADATA min_pat_list_date_dd_mon_rrrr = 01-Jan-2010 min_visit_date_dd_mon_rrrr = 01-Jan-2010 diff --git a/i2p_tasks.py b/i2p_tasks.py index c0cee58..43acc9d 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -82,14 +82,14 @@ class dispensing(I2PScriptTask): script = Script.dispensing def requires(self) -> List[luigi.Task]: - return [encounter()] + return [encounter(), loadRouteMap(), loadUnitMap()] class encounter(I2PScriptTask): script = Script.encounter def requires(self) -> List[luigi.Task]: - return [demographic()] + return [demographic(), loadPayerMap()] class enrollment(I2PScriptTask): @@ -112,14 +112,14 @@ class lab_result_cm(I2PScriptTask): script = Script.lab_result_cm def requires(self) -> List[luigi.Task]: - return [encounter(), loadLabNormal()] + return [encounter(), loadLabNormal(), loadSpecimenSourceMap()] class med_admin(I2PScriptTask): script = Script.med_admin def requires(self) -> List[luigi.Task]: - return [pcornet_init()] + return [pcornet_init(), loadRouteMap(), loadUnitMap()] class obs_clin(I2PScriptTask): @@ -205,11 +205,11 @@ def results(self) -> List[RowProxy]: # TODO: pcornet_init drops and recreates the cdm_status table, forcing all tasks to wait until init is done. # Moving this operation to a distinct task would allow some other tasks (e.g. mapping tasks) to proceed, while init -# performs other labor intesive SQL operations. +# performs other labor intensive SQL operations. # In the mean time, don't forget to make all tasks that use the status table dependent on pcornet_init. # TODO: On a related matter, if the cdm_status table is missing (e.g. on a db where CDM has never been run), running -# the full pipeline, starting at pcornet_loader, will fail. It would be nice to detect this situation and first run -# pcornet_init before attempting other tasks. +# the full pipeline, starting at pcornet_loader, will fail. It would be nice to detect this situation and first +# build the status table before attempting other tasks. class pcornet_init(I2PScriptTask): script = Script.pcornet_init @@ -235,7 +235,7 @@ class prescribing(I2PScriptTask): script = Script.prescribing def requires(self) -> List[luigi.Task]: - return [encounter()] + return [encounter(), loadRouteMap(), loadUnitMap()] class pro_cm(I2PScriptTask): From 8ad8b25850f996ce7205a319267fcd4781f313e5 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 19 Jul 2018 15:56:00 -0500 Subject: [PATCH 430/507] Incorporated POA values not supplied by observation_fact. --- Oracle/diagnosis.sql | 18 ++++++++++++++++-- Oracle/med_admin.sql | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 15e2652..5f7ea2d 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -158,7 +158,21 @@ insert into poafact from i2b2fact factline inner join ENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num inner join pcornet_diag dxsource on factline.modifier_cd = dxsource.c_basecode - and dxsource.c_fullname like '\PCORI_MOD\DX_ORIGIN\BI\DX|BILL:POA\%'; + and dxsource.c_fullname like '\PCORI_MOD\DX_ORIGIN\BI\DX|BILL:POA\%' + + union all + + select patient_num, factline.encounter_num, provider_id, concept_cd, start_date, + case + when sf.tval_char = 'No' then 'N' + when sf.tval_char = 'Unknown' then 'UN' + when sf.tval_char = 'Clinically Undetermined' then 'W' + when sf.tval_char = 'Exempt from POA reporting' then '1' + else 'OT' + end poasource, + sf.tval_char rawpoasource, '@' + from i2b2fact factline + join supplemental_fact sf on factline.instance_num = sf.instance_num and sf.source_column = 'POA_ALT'; execute immediate 'create index poafact_idx on poafact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('POAFACT'); @@ -247,7 +261,7 @@ select distinct factline.patient_num, factline.encounter_num encounterid, enc_ty THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, ':')+1,2),'NI') ELSE null END PDX, CASE WHEN enc_type in ('EI', 'IP') - THEN nvl(SUBSTR(poasource,INSTR(poasource, ':')+1,2),'UN') + THEN nvl(SUBSTR(poasource,INSTR(poasource, ':')+1,2),'NI') ELSE null END DX_POA , rawpoasource RAW_DX_POA from diag_fact_cutoff_filter factline diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index cba71a8..0e3130d 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -111,7 +111,7 @@ select med_start.patient_num , med_start.concept_cd , med_dose.nval_num , med_dose.units_cd - , med_start.modifier_cd -- Modifier rather than raw route. + , med_start.modifier_cd -- Modifier code rather than raw route. from med_start left join BLUEHERONDATA.observation_fact med_dose on med_dose.instance_num = med_start.instance_num From 467676ab01b3b1f61c627a18a6ef94193c2c153f Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 20 Jul 2018 09:13:35 -0500 Subject: [PATCH 431/507] Updating blueherondata references to &&i2b2_data_schema variable. --- Oracle/diagnosis.sql | 2 +- Oracle/dispensing.sql | 6 +++--- Oracle/encounter.sql | 2 +- Oracle/lab_result_cm.sql | 2 +- Oracle/med_admin.sql | 4 ++-- Oracle/prescribing.sql | 16 ++++++++-------- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 5f7ea2d..281123e 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -172,7 +172,7 @@ insert into poafact end poasource, sf.tval_char rawpoasource, '@' from i2b2fact factline - join supplemental_fact sf on factline.instance_num = sf.instance_num and sf.source_column = 'POA_ALT'; + join &&i2b2_data_schema.supplemental_fact sf on factline.instance_num = sf.instance_num and sf.source_column = 'POA_ALT'; execute immediate 'create index poafact_idx on poafact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('POAFACT'); diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index 86b2050..893a286 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -134,7 +134,7 @@ with disp_status as ( select encounter_num , instance_num , to_number(tval_char) dose - from blueherondata.supplemental_fact + from &&i2b2_data_schema.supplemental_fact where source_column = 'DISCRETE_DOSE' ) , disp_unit as ( @@ -142,7 +142,7 @@ with disp_status as ( , sf.instance_num , nvl(um.code, 'OT') code , sf.tval_char - from blueherondata.supplemental_fact sf + from &&i2b2_data_schema.supplemental_fact sf left join unit_map um on sf.tval_char = um.unit_name where sf.source_column = 'DOSE_UNITS' ) @@ -151,7 +151,7 @@ with disp_status as ( , sf.instance_num , rm.code , sf.tval_char - from blueherondata.supplemental_fact sf + from &&i2b2_data_schema.supplemental_fact sf left join route_map rm on lower(sf.tval_char) = lower(rm.route_name) where sf.source_column = 'ADMIN_ROUTE' ) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 0c7f5cc..fe77b64 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -91,7 +91,7 @@ select f.encounter_num , sf.tval_char raw_payer_type_primary from i2b2fact f join demographic d on f.patient_num = d.patid -left join blueherondata.supplemental_fact sf on f.instance_num = sf.instance_num +left join &&i2b2_data_schema.supplemental_fact sf on f.instance_num = sf.instance_num left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) and lower(pm.financial_class) = lower(sf.tval_char) where concept_cd like 'O2|PAYER_PRIMARY:%' or concept_cd like 'IDX|PAYER_PRIMARY:%' and sf.source_column = 'FINANCIAL_CLASS' diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index dffaf0c..5a19c96 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -67,7 +67,7 @@ select lab_result_cm_seq.nextval LAB_RESULT_CM_ID , valueflag_cd ABN_IND , valtype_cd RAW_RESULT , concept_cd RAW_FACILITY_CODE -from blueherondata.observation_fact m +from &&i2b2_data_schema.observation_fact m join encounter enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num where concept_cd between 'KUH|COMPONENT_ID:' and 'KUH|COMPONENT_ID:~' and modifier_cd in ('@') -- exclude analyitics: Labs|Aggregate:Median, ... diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 0e3130d..7db509a 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -70,7 +70,7 @@ insert into med_admin(patid , raw_medadmin_route) with med_start as ( select patient_num, encounter_num, provider_id, start_date, end_date, concept_cd, modifier_cd, instance_num - from BLUEHERONDATA.observation_fact + from &&i2b2_data_schema.observation_fact where modifier_cd = 'MedObs|MAR:New Bag' or modifier_cd = 'MedObs|MAR:Downtime Given - New Bag' or modifier_cd = 'MedObs|MAR:Given - Without Order' @@ -113,7 +113,7 @@ select med_start.patient_num , med_dose.units_cd , med_start.modifier_cd -- Modifier code rather than raw route. from med_start -left join BLUEHERONDATA.observation_fact med_dose +left join &&i2b2_data_schema.observation_fact med_dose on med_dose.instance_num = med_start.instance_num and med_dose.start_date = med_start.start_date and (med_dose.modifier_cd = 'MedObs:MAR_Dose|puff' diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 8eb6640..1bc9342 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -88,7 +88,7 @@ select cast(prescribing_seq.nextval as varchar(19)) prescribingid , case when trim(translate(tval_char, '0123456789.', ' ')) is null then um.code else null end rx_dose_ordered_unit , tval_char raw_rx_dose_ordered , units_cd raw_rx_dose_ordered_unit -from blueherondata.observation_fact rx +from &&i2b2_data_schema.observation_fact rx join encounter en on rx.encounter_num = en.encounterid join unit_map um on rx.units_cd = um.unit_name where rx.modifier_cd in ('MedObs:Inpatient', 'MedObs:Outpatient') @@ -135,7 +135,7 @@ left join (select instance_num , concept_cd , nval_num - from blueherondata.observation_fact + from &&i2b2_data_schema.observation_fact where modifier_cd = 'RX_REFILLS' /* aka: select c_basecode from pcornet_med refillscode @@ -151,7 +151,7 @@ left join (select instance_num , concept_cd , pcori_basecode - from blueherondata.observation_fact + from &&i2b2_data_schema.observation_fact join pcornet_med on modifier_cd = c_basecode and c_fullname like '\PCORI_MOD\RX_FREQUENCY\%' ) freq on freq.instance_num = rx.instance_num and freq.concept_cd = rx.concept_cd @@ -165,7 +165,7 @@ left join (select instance_num , concept_cd , nval_num - from blueherondata.observation_fact + from &&i2b2_data_schema.observation_fact where modifier_cd = 'RX_QUANTITY' /* aka: select c_basecode from pcornet_med refillscode @@ -181,7 +181,7 @@ left join (select instance_num , concept_cd , nval_num - from blueherondata.observation_fact + from &&i2b2_data_schema.observation_fact where modifier_cd = 'RX_DAYS_SUPPLY' /* aka: select c_basecode from pcornet_med refillscode @@ -197,7 +197,7 @@ left join (select instance_num , concept_cd , pcori_basecode - from blueherondata.observation_fact + from &&i2b2_data_schema.observation_fact join pcornet_med on modifier_cd = c_basecode and c_fullname like '\PCORI_MOD\RX_BASIS\%' and modifier_cd in ('MedObs:Inpatient', 'MedObs:Outpatient') @@ -212,7 +212,7 @@ left join (select instance_num , concept_cd , 'YES' as tval_char - from blueherondata.observation_fact + from &&i2b2_data_schema.observation_fact where modifier_cd = 'MedObs:PRN' /* aka: select c_basecode from pcornet_med code @@ -228,7 +228,7 @@ from prescribing_w_prn rx left join (select instance_num , tval_char - from blueherondata.supplemental_fact + from &&i2b2_data_schema.supplemental_fact where source_column = 'PRESCRIBING_ROUTE' ) rt on rt.instance_num = rx.instance_num left join route_map rm on lower(rt.tval_char) = lower(rm.route_name) From a2f6a966bc4be78ce5f9c0377f1cf1eaede9505a Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 20 Jul 2018 11:18:54 -0500 Subject: [PATCH 432/507] IDX payer data in encounter, dispense as written in prescribing. --- Oracle/encounter.sql | 18 +++++++++-- Oracle/prescribing.sql | 16 ++++++++-- curated_data/payer_map.csv | 62 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 4 deletions(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index fe77b64..7a4c989 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -81,8 +81,8 @@ insert into encounter(PATID, ENCOUNTERID, admit_date, ADMIT_TIME, DISCHARGE_DATE , RAW_DISCHARGE_DISPOSITION, RAW_DISCHARGE_STATUS, RAW_DRG_TYPE, RAW_ADMITTING_SOURCE, RAW_FACILITY_TYPE , RAW_PAYER_TYPE_PRIMARY, RAW_PAYER_NAME_PRIMARY, RAW_PAYER_ID_PRIMARY, RAW_PAYER_TYPE_SECONDARY , RAW_PAYER_NAME_SECONDARY, RAW_PAYER_ID_SECONDARY) + with payer as ( --- TODO: Nulls and IDX. select f.encounter_num , f.patient_num , pm.code payer_type_primary @@ -93,9 +93,23 @@ from i2b2fact f join demographic d on f.patient_num = d.patid left join &&i2b2_data_schema.supplemental_fact sf on f.instance_num = sf.instance_num left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) and lower(pm.financial_class) = lower(sf.tval_char) -where concept_cd like 'O2|PAYER_PRIMARY:%' or concept_cd like 'IDX|PAYER_PRIMARY:%' +where concept_cd like 'O2|PAYER_PRIMARY:%' and sf.source_column = 'FINANCIAL_CLASS' + +union all + +select f.encounter_num + , f.patient_num + , pm.code payer_type_primary + ,f.tval_char raw_payer_name_primary + , SUBSTR(f.concept_cd, INSTR(f.concept_cd, ':', 1, 1) + 1) raw_payer_id_primary + , null raw_payer_type_primary +from i2b2fact f +join demographic d on f.patient_num = d.patid +left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) +where concept_cd like 'IDX|PAYER_PRIMARY:%' ) + select distinct v.patient_num, v.encounter_num, start_Date, diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 1bc9342..e6522d3 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -234,6 +234,18 @@ left join left join route_map rm on lower(rt.tval_char) = lower(rm.route_name) / +create table prescribing_w_daw as +select rx.* +, nvl(tval_char, 'NI') rx_dispense_as_written +from prescribing_w_route rx +left join + (select instance_num + , tval_char + from &&i2b2_data_schema.supplemental_fact + where source_column = 'DISPENSE_AS_WRITTEN' + ) daw on daw.instance_num = rx.instance_num +/ + create table prescribing as select rx.prescribingid , rx.patid @@ -255,7 +267,7 @@ select rx.prescribingid , decode(rx.modifier_cd, 'MedObs:Inpatient', '01', 'MedObs:Outpatient', '02') rx_basis , rx.rxnorm_cui , 'OD' rx_source -, cast(null as varchar(2)) rx_dispense_as_written +, rx.rx_dispense_as_written , rx.raw_rx_med_name , cast(null as varchar(50)) raw_rx_frequency , rx.raw_rxnorm_cui @@ -268,7 +280,7 @@ select rx.prescribingid /* ISSUE: HERON should have an actual order time. idea: store real difference between order date start data, possibly using the update date */ -from prescribing_w_route rx +from prescribing_w_daw rx / create index prescribing_idx on prescribing (PATID, ENCOUNTERID) diff --git a/curated_data/payer_map.csv b/curated_data/payer_map.csv index 9ee58bf..0273654 100644 --- a/curated_data/payer_map.csv +++ b/curated_data/payer_map.csv @@ -303,3 +303,65 @@ GENERIC WORKERS COMP,Worker's Comp,95,Worker's Compensation MS WORKERS COMP,Worker's Comp,95,Worker's Compensation WORKERS COMP,Worker's Comp,95,Worker's Compensation WORKMAN'S COMPENSATION,Worker's Comp,95,Worker's Compensation +AIH COLLECTIONS,,349,Other +AMERIGROUP KS A,,38,"Other Government (Federal, State, Local not specified)" +AMERIGROUP KS B,,38,"Other Government (Federal, State, Local not specified)" +BAD DEBT,,349,Other +BANKRUPTCY,,349,Other +BCBS KC APPEALS-KUPI,,6,BLUE CROSS/BLUE SHIELD +BCBS KC APPEALS-PER SE,,6,BLUE CROSS/BLUE SHIELD +BCBS KC PREFERRED CARE OTHER,,6,BLUE CROSS/BLUE SHIELD +BCBS KS APPEALS-KUPI,,6,BLUE CROSS/BLUE SHIELD +BCBS KS APPEALS-PER SE,,6,BLUE CROSS/BLUE SHIELD +BCBS KS HMO/POS A,,6,BLUE CROSS/BLUE SHIELD +BERLIN WHEELER COLLECTIONS,,349,Other +BERMAN & RABIN COLLECTIONS,,349,Other +BUDGET PLAN,,349,Other +CHARITY/HARDSHIP,,821,Charity +COLLECTION AGENCY,,349,Other +CONSUMER COLL MGMT COLLECTIONS,,349,Other +CORPORATE ACCOUNTS,,349,Other +CREDIT LETTER SENT,,349,Other +CRIME VICTIMS MO,,349,Other +CSR COLLECTIONS,,349,Other +DENTAL A,,561,Dental +DO NOT BILL,,349,Other +DO NOT DUNN,,349,Other +ELECTIVE PACKAGE RATE A,,349,Other +FHP APPEAL-PER SE,,349,Other +FHP APPEALS-KUPI,,349,Other +FOREIGN MAIL,,349,Other +KANCARE BEHAVIORAL HLTH A,,38,"Other Government (Federal, State, Local not specified)" +KANCARE BEHAVIORAL HLTH B,,38,"Other Government (Federal, State, Local not specified)" +KUPI ADMINISTRATIVE,,349,Other +KUPI COMPLIANCE,,349,Other +LITIGATION/LEGAL,,349,Other +LIVING DONOR BALANCE,,349,Other +MEDICAID KS APPEALS-KUPI,,2,MEDICAID +MEDICAID KS APPEALS-PER SE,,2,MEDICAID +MEDICAID KS HMO,,2,MEDICAID +MEDICAID KS HMO (102 REPLACEMENT),,2,MEDICAID +MEDICAID MO APPEALS-KUPI,,2,MEDICAID +MEDICAID MO APPEALS-PER SE,,2,MEDICAID +MEDICAID MO HMO B,,2,MEDICAID +MEDICARE APPEALS-KUPI,,1,MEDICARE +MEDICARE APPEALS-PER SE,,1,MEDICARE +MEDICARE RR APPEALS-KUPI,,1,MEDICARE +MEDICARE RR APPEALS-PER SE,,1,MEDICARE +OTHER/COMMERCIAL APPEALS-KUPI,,349,Other +OTHER/COMMERCIAL APPEALS-PER SE,,349,Other +PATIENT REQUEST-DO NOT BILL INSURANCE,,349,Other +PER SE TURN LIST REVIEW,,349,Other +RECONCILIATION,,349,Other +RETURN MAIL/BAD ADDRESS,,349,Other +RTN MAIL AFTER TURN LIST RVW,,349,Other +SUNFLOWER STATE KS A,,2,MEDICAID +SUNFLOWER STATE KS B,,2,MEDICAID +SUSPENDED CLAIMS,,349,Other +TRICARE UHC HMO A,,3111,TRICARE Prime?HMO +TRICARE UHC HMO B,,3111,TRICARE Prime?HMO +TRICARE UHC PPO A,,3112,TRICARE Extra?PPO +TRICARE UHC PPO B,,3112,TRICARE Extra?PPO +TURN LIST/CREDIT REVIEW,,349,Other +UHC COMMUNITY PLAN KS A,,5,PRIVATE HEALTH INSURANCE +UHC COMMUNITY PLAN KS B,,5,PRIVATE HEALTH INSURANCE From 6fd25f52d6f98a7cabd45f12571f60c6fa41d0f4 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 20 Jul 2018 13:42:30 -0500 Subject: [PATCH 433/507] Temporarily disabling IDX payer for troubleshooting. --- Oracle/encounter.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 7a4c989..5181e3e 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -96,6 +96,7 @@ left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) and lower(pm where concept_cd like 'O2|PAYER_PRIMARY:%' and sf.source_column = 'FINANCIAL_CLASS' +/* union all select f.encounter_num @@ -108,6 +109,7 @@ from i2b2fact f join demographic d on f.patient_num = d.patid left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) where concept_cd like 'IDX|PAYER_PRIMARY:%' +*/ ) select distinct v.patient_num, From e6554cf78634aa110dbc548240985e17b5343bd4 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 20 Jul 2018 13:56:50 -0500 Subject: [PATCH 434/507] Adding drop to cleanup old prescribing_w_daw table. --- Oracle/prescribing.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index e6522d3..3369078 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -43,6 +43,10 @@ PMN_DROPSQL('drop table prescribing_w_route'); END; / BEGIN +PMN_DROPSQL('drop table prescribing_w_daw'); +END; +/ +BEGIN PMN_DROPSQL('DROP sequence prescribing_seq'); END; / From 7db3ac6f4066460330103f4b63b11a12a93be590 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 20 Jul 2018 15:48:56 -0500 Subject: [PATCH 435/507] Optimization of encouter query, create enctype table. The encounter query was running for more than 6 hours in B2, versus just under 2 hours in A1. The enctype subquery seemed to be the contributing factor. --- Oracle/encounter.sql | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 5181e3e..a56ddbd 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -52,15 +52,28 @@ CREATE TABLE drg ( RN NUMBER ) / +BEGIN +PMN_DROPSQL('DROP TABLE enctype'); +END; +/ +CREATE TABLE enctype ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + INOUT_CD VARCHAR2(50), + PCORI_ENCTYPE VARCHAR2(3), +) +/ create or replace procedure PCORNetEncounter as begin PMN_DROPSQL('drop index encounter_pk'); PMN_DROPSQL('drop index encounter_idx'); PMN_DROPSQL('drop index drg_idx'); +PMN_DROPSQL('drop index enctype_idx'); execute immediate 'truncate table encounter'; execute immediate 'truncate table drg'; +execute immediate 'truncate table enctype'; insert into drg select * from @@ -75,6 +88,14 @@ where rn=1; execute immediate 'create index drg_idx on drg (patient_num, encounter_num)'; --GATHER_TABLE_STATS('drg'); +-- Encounter type. Note that this requires a full table scan on the ontology table, so it is not particularly efficient. +insert into enctype +select patient_num, encounter_num, inout_cd,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,2) pcori_enctype +from pcornet_cdm.i2b2visit v +inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%'; + +execute immediate 'create index enctype_idx on enc_type (patient_num, encounter_num)'; + insert into encounter(PATID, ENCOUNTERID, admit_date, ADMIT_TIME, DISCHARGE_DATE, DISCHARGE_TIME, PROVIDERID , FACILITY_LOCATION, ENC_TYPE, FACILITYID, DISCHARGE_DISPOSITION, DISCHARGE_STATUS, DRG, DRG_TYPE , ADMITTING_SOURCE, PAYER_TYPE_PRIMARY, PAYER_TYPE_SECONDARY, FACILITY_TYPE, RAW_SITEID, RAW_ENC_TYPE @@ -145,10 +166,7 @@ select distinct v.patient_num, from i2b2visit v inner join demographic d on v.patient_num=d.patid left outer join drg -- This section is bugfixed to only include 1 drg if multiple DRG types exist in a single encounter... on drg.patient_num=v.patient_num and drg.encounter_num=v.encounter_num -left outer join --- Encounter type. Note that this requires a full table scan on the ontology table, so it is not particularly efficient. -(select patient_num, encounter_num, inout_cd,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,2) pcori_enctype from i2b2visit v - inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype +left outer join enctype on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num left outer join payer p on p.patient_num = v.patient_num and p.encounter_num = v.encounter_num ; From edd8d593d3a4991354bcc66b5b086ca724a2ce2b Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 20 Jul 2018 15:54:26 -0500 Subject: [PATCH 436/507] Too many commas. --- Oracle/encounter.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index a56ddbd..e8f038a 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -60,7 +60,7 @@ CREATE TABLE enctype ( PATIENT_NUM NUMBER(38) NOT NULL, ENCOUNTER_NUM NUMBER(38) NOT NULL, INOUT_CD VARCHAR2(50), - PCORI_ENCTYPE VARCHAR2(3), + PCORI_ENCTYPE VARCHAR2(3) ) / create or replace procedure PCORNetEncounter as From f15668f4d7594f6328829d804473350bc79d277e Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 20 Jul 2018 15:58:01 -0500 Subject: [PATCH 437/507] Too many underscores. --- Oracle/encounter.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index e8f038a..355e230 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -94,7 +94,7 @@ select patient_num, encounter_num, inout_cd,SUBSTR(pcori_basecode,INSTR(pcori_ba from pcornet_cdm.i2b2visit v inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%'; -execute immediate 'create index enctype_idx on enc_type (patient_num, encounter_num)'; +execute immediate 'create index enctype_idx on enctype (patient_num, encounter_num)'; insert into encounter(PATID, ENCOUNTERID, admit_date, ADMIT_TIME, DISCHARGE_DATE, DISCHARGE_TIME, PROVIDERID , FACILITY_LOCATION, ENC_TYPE, FACILITYID, DISCHARGE_DISPOSITION, DISCHARGE_STATUS, DRG, DRG_TYPE From 2a694b48114598f09f413a1e88df316e44d30c8f Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 23 Jul 2018 09:02:52 -0500 Subject: [PATCH 438/507] Temporary rollback of encounter to establish baseline on B2. --- Oracle/encounter.sql | 116 +++++++------------------------------------ 1 file changed, 18 insertions(+), 98 deletions(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 355e230..f581993 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -22,22 +22,12 @@ CREATE TABLE encounter( DRG varchar(3) NULL, DRG_TYPE varchar(2) NULL, ADMITTING_SOURCE varchar(2) NULL, - PAYER_TYPE_PRIMARY varchar(5) NULL, - PAYER_TYPE_SECONDARY varchar(5) NULL, - FACILITY_TYPE varchar(50) NULL, RAW_SITEID varchar (50) NULL, RAW_ENC_TYPE varchar(50) NULL, RAW_DISCHARGE_DISPOSITION varchar(50) NULL, RAW_DISCHARGE_STATUS varchar(50) NULL, RAW_DRG_TYPE varchar(50) NULL, - RAW_ADMITTING_SOURCE varchar(50) NULL, - RAW_FACILITY_TYPE varchar(50) NULL, - RAW_PAYER_TYPE_PRIMARY varchar(50) NULL, - RAW_PAYER_NAME_PRIMARY varchar(50) NULL, - RAW_PAYER_ID_PRIMARY varchar(50) NULL, - RAW_PAYER_TYPE_SECONDARY varchar(50) NULL, - RAW_PAYER_NAME_SECONDARY varchar(50) NULL, - RAW_PAYER_ID_SECONDARY varchar(50) NULL + RAW_ADMITTING_SOURCE varchar(50) NULL ) / BEGIN @@ -52,28 +42,15 @@ CREATE TABLE drg ( RN NUMBER ) / -BEGIN -PMN_DROPSQL('DROP TABLE enctype'); -END; -/ -CREATE TABLE enctype ( - PATIENT_NUM NUMBER(38) NOT NULL, - ENCOUNTER_NUM NUMBER(38) NOT NULL, - INOUT_CD VARCHAR2(50), - PCORI_ENCTYPE VARCHAR2(3) -) -/ create or replace procedure PCORNetEncounter as begin PMN_DROPSQL('drop index encounter_pk'); PMN_DROPSQL('drop index encounter_idx'); PMN_DROPSQL('drop index drg_idx'); -PMN_DROPSQL('drop index enctype_idx'); execute immediate 'truncate table encounter'; execute immediate 'truncate table drg'; -execute immediate 'truncate table enctype'; insert into drg select * from @@ -88,88 +65,31 @@ where rn=1; execute immediate 'create index drg_idx on drg (patient_num, encounter_num)'; --GATHER_TABLE_STATS('drg'); --- Encounter type. Note that this requires a full table scan on the ontology table, so it is not particularly efficient. -insert into enctype -select patient_num, encounter_num, inout_cd,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,2) pcori_enctype -from pcornet_cdm.i2b2visit v -inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%'; - -execute immediate 'create index enctype_idx on enctype (patient_num, encounter_num)'; - -insert into encounter(PATID, ENCOUNTERID, admit_date, ADMIT_TIME, DISCHARGE_DATE, DISCHARGE_TIME, PROVIDERID - , FACILITY_LOCATION, ENC_TYPE, FACILITYID, DISCHARGE_DISPOSITION, DISCHARGE_STATUS, DRG, DRG_TYPE - , ADMITTING_SOURCE, PAYER_TYPE_PRIMARY, PAYER_TYPE_SECONDARY, FACILITY_TYPE, RAW_SITEID, RAW_ENC_TYPE - , RAW_DISCHARGE_DISPOSITION, RAW_DISCHARGE_STATUS, RAW_DRG_TYPE, RAW_ADMITTING_SOURCE, RAW_FACILITY_TYPE - , RAW_PAYER_TYPE_PRIMARY, RAW_PAYER_NAME_PRIMARY, RAW_PAYER_ID_PRIMARY, RAW_PAYER_TYPE_SECONDARY - , RAW_PAYER_NAME_SECONDARY, RAW_PAYER_ID_SECONDARY) - -with payer as ( -select f.encounter_num - , f.patient_num - , pm.code payer_type_primary - ,f.tval_char raw_payer_name_primary - , SUBSTR(f.concept_cd, INSTR(f.concept_cd, ':', 1, 1) + 1) raw_payer_id_primary - , sf.tval_char raw_payer_type_primary -from i2b2fact f -join demographic d on f.patient_num = d.patid -left join &&i2b2_data_schema.supplemental_fact sf on f.instance_num = sf.instance_num -left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) and lower(pm.financial_class) = lower(sf.tval_char) -where concept_cd like 'O2|PAYER_PRIMARY:%' -and sf.source_column = 'FINANCIAL_CLASS' - -/* -union all - -select f.encounter_num - , f.patient_num - , pm.code payer_type_primary - ,f.tval_char raw_payer_name_primary - , SUBSTR(f.concept_cd, INSTR(f.concept_cd, ':', 1, 1) + 1) raw_payer_id_primary - , null raw_payer_type_primary -from i2b2fact f -join demographic d on f.patient_num = d.patid -left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) -where concept_cd like 'IDX|PAYER_PRIMARY:%' -*/ -) - -select distinct v.patient_num, - v.encounter_num, +insert into encounter(PATID,ENCOUNTERID,admit_date ,ADMIT_TIME , + DISCHARGE_DATE ,DISCHARGE_TIME ,PROVIDERID ,FACILITY_LOCATION + ,ENC_TYPE ,FACILITYID ,DISCHARGE_DISPOSITION , + DISCHARGE_STATUS ,DRG ,DRG_TYPE ,ADMITTING_SOURCE) +select distinct v.patient_num, v.encounter_num, start_Date, to_char(start_Date,'HH24:MI'), end_Date, to_char(end_Date,'HH24:MI'), providerid, - 'NI' location_zip, /* See TODO above */ - (case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, - 'NI' facility_id, /* See TODO above */ - CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END, - CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END, - drg.drg, drg_type, - CASE WHEN admitting_source IS NULL THEN 'NI' ELSE admitting_source END admitting_source, - p.payer_type_primary, - null payer_type_secondary, - null facility_type, - null raw_siteid, - null raw_enc_type, - null raw_discharge_disposition, - null raw_discharge_status, - null raw_drg_type, - null raw_admitting_source, - null raw_facility_type, - p.raw_payer_type_primary, - p.raw_payer_name_primary, - p.raw_payer_id_primary, - null raw_payer_type_secondary, - null raw_payer_name_secondary, - null raw_payer_id_secondary + 'NI' location_zip, /* See TODO above */ +(case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, + 'NI' facility_id, /* See TODO above */ + CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END, + CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END, + drg.drg, drg_type, + CASE WHEN admitting_source IS NULL THEN 'NI' ELSE admitting_source END admitting_source from i2b2visit v inner join demographic d on v.patient_num=d.patid left outer join drg -- This section is bugfixed to only include 1 drg if multiple DRG types exist in a single encounter... on drg.patient_num=v.patient_num and drg.encounter_num=v.encounter_num -left outer join enctype - on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num -left outer join payer p on p.patient_num = v.patient_num and p.encounter_num = v.encounter_num -; +left outer join +-- Encounter type. Note that this requires a full table scan on the ontology table, so it is not particularly efficient. +(select patient_num, encounter_num, inout_cd,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,2) pcori_enctype from i2b2visit v + inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype + on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num; execute immediate 'create unique index encounter_pk on encounter (ENCOUNTERID)'; execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; From a592cc153dfc4fa8cc1ee4f9870305bd13322a0a Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 23 Jul 2018 13:40:18 -0500 Subject: [PATCH 439/507] Test code for performance on B2. --- Oracle/encounter.sql | 97 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 16 deletions(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index f581993..cb9b3e0 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -22,12 +22,22 @@ CREATE TABLE encounter( DRG varchar(3) NULL, DRG_TYPE varchar(2) NULL, ADMITTING_SOURCE varchar(2) NULL, + PAYER_TYPE_PRIMARY varchar(5) NULL, + PAYER_TYPE_SECONDARY varchar(5) NULL, + FACILITY_TYPE varchar(50) NULL, RAW_SITEID varchar (50) NULL, RAW_ENC_TYPE varchar(50) NULL, RAW_DISCHARGE_DISPOSITION varchar(50) NULL, RAW_DISCHARGE_STATUS varchar(50) NULL, RAW_DRG_TYPE varchar(50) NULL, - RAW_ADMITTING_SOURCE varchar(50) NULL + RAW_ADMITTING_SOURCE varchar(50) NULL, + RAW_FACILITY_TYPE varchar(50) NULL, + RAW_PAYER_TYPE_PRIMARY varchar(50) NULL, + RAW_PAYER_NAME_PRIMARY varchar(50) NULL, + RAW_PAYER_ID_PRIMARY varchar(50) NULL, + RAW_PAYER_TYPE_SECONDARY varchar(50) NULL, + RAW_PAYER_NAME_SECONDARY varchar(50) NULL, + RAW_PAYER_ID_SECONDARY varchar(50) NULL ) / BEGIN @@ -65,23 +75,75 @@ where rn=1; execute immediate 'create index drg_idx on drg (patient_num, encounter_num)'; --GATHER_TABLE_STATS('drg'); -insert into encounter(PATID,ENCOUNTERID,admit_date ,ADMIT_TIME , - DISCHARGE_DATE ,DISCHARGE_TIME ,PROVIDERID ,FACILITY_LOCATION - ,ENC_TYPE ,FACILITYID ,DISCHARGE_DISPOSITION , - DISCHARGE_STATUS ,DRG ,DRG_TYPE ,ADMITTING_SOURCE) -select distinct v.patient_num, v.encounter_num, +insert into encounter(PATID + , ENCOUNTERID + , admit_date + , ADMIT_TIME + , DISCHARGE_DATE + , DISCHARGE_TIME + , PROVIDERID + , FACILITY_LOCATION + , ENC_TYPE + , FACILITYID + , DISCHARGE_DISPOSITION + , DISCHARGE_STATUS + , DRG + , DRG_TYPE + , ADMITTING_SOURCE + , PAYER_TYPE_PRIMARY + , RAW_PAYER_TYPE_PRIMARY + , RAW_PAYER_NAME_PRIMARY + , RAW_PAYER_ID_PRIMARY) + +with payer as ( + select f.encounter_num + , f.patient_num + , pm.code payer_type_primary + ,f.tval_char raw_payer_name_primary + , SUBSTR(f.concept_cd, INSTR(f.concept_cd, ':', 1, 1) + 1) raw_payer_id_primary + , sf.tval_char raw_payer_type_primary + from i2b2fact f + join demographic d on f.patient_num = d.patid + left join blueherondata.supplemental_fact sf on f.instance_num = sf.instance_num + left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) and lower(pm.financial_class) = lower(sf.tval_char) + where concept_cd like 'O2|PAYER_PRIMARY:%' + and sf.source_column = 'FINANCIAL_CLASS' + +/* + union all + + select f.encounter_num + , f.patient_num + , pm.code payer_type_primary + ,f.tval_char raw_payer_name_primary + , SUBSTR(f.concept_cd, INSTR(f.concept_cd, ':', 1, 1) + 1) raw_payer_id_primary + , null raw_payer_type_primary + from i2b2fact f + join demographic d on f.patient_num = d.patid + left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) + where concept_cd like 'IDX|PAYER_PRIMARY:%' +*/ +) + +select v.patient_num, + v.encounter_num, start_Date, to_char(start_Date,'HH24:MI'), end_Date, to_char(end_Date,'HH24:MI'), providerid, - 'NI' location_zip, /* See TODO above */ -(case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, - 'NI' facility_id, /* See TODO above */ - CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END, - CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END, - drg.drg, drg_type, - CASE WHEN admitting_source IS NULL THEN 'NI' ELSE admitting_source END admitting_source + 'NI' location_zip, /* See TODO above */ + (case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, + 'NI' facility_id, /* See TODO above */ + CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END, + CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END, + drg.drg, + drg_type, + CASE WHEN admitting_source IS NULL THEN 'NI' ELSE admitting_source END admitting_source, + p.payer_type_primary, + p.raw_payer_type_primary, + p.raw_payer_name_primary, + p.raw_payer_id_primary from i2b2visit v inner join demographic d on v.patient_num=d.patid left outer join drg -- This section is bugfixed to only include 1 drg if multiple DRG types exist in a single encounter... on drg.patient_num=v.patient_num and drg.encounter_num=v.encounter_num @@ -89,10 +151,13 @@ left outer join -- Encounter type. Note that this requires a full table scan on the ontology table, so it is not particularly efficient. (select patient_num, encounter_num, inout_cd,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,2) pcori_enctype from i2b2visit v inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype - on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num; + on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num +left outer join payer p on p.patient_num = v.patient_num and p.encounter_num = v.encounter_num +; -execute immediate 'create unique index encounter_pk on encounter (ENCOUNTERID)'; -execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; +-- TODO : Re-enable indices!!!! +--execute immediate 'create unique index encounter_pk on encounter (ENCOUNTERID)'; +--execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; GATHER_TABLE_STATS('ENCOUNTER'); end PCORNetEncounter; From 7d008781acfdc048e6d1839fca40d948600144eb Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 23 Jul 2018 15:21:13 -0500 Subject: [PATCH 440/507] A desperate attemtp to isolate performance issues on B2. --- Oracle/encounter.sql | 46 ++++++++------------------------------------ 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index cb9b3e0..cd4a5e7 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -95,37 +95,7 @@ insert into encounter(PATID , RAW_PAYER_NAME_PRIMARY , RAW_PAYER_ID_PRIMARY) -with payer as ( - select f.encounter_num - , f.patient_num - , pm.code payer_type_primary - ,f.tval_char raw_payer_name_primary - , SUBSTR(f.concept_cd, INSTR(f.concept_cd, ':', 1, 1) + 1) raw_payer_id_primary - , sf.tval_char raw_payer_type_primary - from i2b2fact f - join demographic d on f.patient_num = d.patid - left join blueherondata.supplemental_fact sf on f.instance_num = sf.instance_num - left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) and lower(pm.financial_class) = lower(sf.tval_char) - where concept_cd like 'O2|PAYER_PRIMARY:%' - and sf.source_column = 'FINANCIAL_CLASS' - -/* - union all - - select f.encounter_num - , f.patient_num - , pm.code payer_type_primary - ,f.tval_char raw_payer_name_primary - , SUBSTR(f.concept_cd, INSTR(f.concept_cd, ':', 1, 1) + 1) raw_payer_id_primary - , null raw_payer_type_primary - from i2b2fact f - join demographic d on f.patient_num = d.patid - left join payer_map pm on lower(pm.payer_name) = lower(f.tval_char) - where concept_cd like 'IDX|PAYER_PRIMARY:%' -*/ -) - -select v.patient_num, +select distinct v.patient_num, v.encounter_num, start_Date, to_char(start_Date,'HH24:MI'), @@ -140,10 +110,10 @@ select v.patient_num, drg.drg, drg_type, CASE WHEN admitting_source IS NULL THEN 'NI' ELSE admitting_source END admitting_source, - p.payer_type_primary, - p.raw_payer_type_primary, - p.raw_payer_name_primary, - p.raw_payer_id_primary + null, + null, + null, + null from i2b2visit v inner join demographic d on v.patient_num=d.patid left outer join drg -- This section is bugfixed to only include 1 drg if multiple DRG types exist in a single encounter... on drg.patient_num=v.patient_num and drg.encounter_num=v.encounter_num @@ -152,12 +122,12 @@ left outer join (select patient_num, encounter_num, inout_cd,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,2) pcori_enctype from i2b2visit v inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num -left outer join payer p on p.patient_num = v.patient_num and p.encounter_num = v.encounter_num +--left outer join payer p on p.patient_num = v.patient_num and p.encounter_num = v.encounter_num ; -- TODO : Re-enable indices!!!! ---execute immediate 'create unique index encounter_pk on encounter (ENCOUNTERID)'; ---execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; +execute immediate 'create unique index encounter_pk on encounter (ENCOUNTERID)'; +execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; GATHER_TABLE_STATS('ENCOUNTER'); end PCORNetEncounter; From 7eab670d2401448381c35a8f7b3e4529fa3e8be5 Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 24 Jul 2018 15:34:18 -0500 Subject: [PATCH 441/507] Optimized encounter script. The real performance hit came on the observation_fact join. Run time on B2 jumped from 15 minutes to 15 hours. Isolating ob_fact reduced runtime to 1 hour. --- Oracle/encounter.sql | 234 +++++++++++++++++++++++-------------------- 1 file changed, 124 insertions(+), 110 deletions(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index cd4a5e7..eea2898 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -2,67 +2,38 @@ */ insert into cdm_status (task, start_time) select 'encounter', sysdate from dual / + BEGIN PMN_DROPSQL('DROP TABLE encounter'); END; / -CREATE TABLE encounter( - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NOT NULL, - ADMIT_DATE date NULL, - ADMIT_TIME varchar(5) NULL, - DISCHARGE_DATE date NULL, - DISCHARGE_TIME varchar(5) NULL, - PROVIDERID varchar(50) NULL, - FACILITY_LOCATION varchar(3) NULL, - ENC_TYPE varchar(2) NOT NULL, - FACILITYID varchar(50) NULL, - DISCHARGE_DISPOSITION varchar(2) NULL, - DISCHARGE_STATUS varchar(2) NULL, - DRG varchar(3) NULL, - DRG_TYPE varchar(2) NULL, - ADMITTING_SOURCE varchar(2) NULL, - PAYER_TYPE_PRIMARY varchar(5) NULL, - PAYER_TYPE_SECONDARY varchar(5) NULL, - FACILITY_TYPE varchar(50) NULL, - RAW_SITEID varchar (50) NULL, - RAW_ENC_TYPE varchar(50) NULL, - RAW_DISCHARGE_DISPOSITION varchar(50) NULL, - RAW_DISCHARGE_STATUS varchar(50) NULL, - RAW_DRG_TYPE varchar(50) NULL, - RAW_ADMITTING_SOURCE varchar(50) NULL, - RAW_FACILITY_TYPE varchar(50) NULL, - RAW_PAYER_TYPE_PRIMARY varchar(50) NULL, - RAW_PAYER_NAME_PRIMARY varchar(50) NULL, - RAW_PAYER_ID_PRIMARY varchar(50) NULL, - RAW_PAYER_TYPE_SECONDARY varchar(50) NULL, - RAW_PAYER_NAME_SECONDARY varchar(50) NULL, - RAW_PAYER_ID_SECONDARY varchar(50) NULL -) -/ + BEGIN PMN_DROPSQL('DROP TABLE drg'); END; / -CREATE TABLE drg ( - PATIENT_NUM NUMBER(38) NOT NULL, - ENCOUNTER_NUM NUMBER(38) NOT NULL, - DRG_TYPE VARCHAR2(2), - DRG VARCHAR2(3), - RN NUMBER -) + +BEGIN +PMN_DROPSQL('drop table encounter_key'); +END; / -create or replace procedure PCORNetEncounter as -begin -PMN_DROPSQL('drop index encounter_pk'); -PMN_DROPSQL('drop index encounter_idx'); -PMN_DROPSQL('drop index drg_idx'); +BEGIN +PMN_DROPSQL('drop table encounter_w_drg'); +END; +/ -execute immediate 'truncate table encounter'; -execute immediate 'truncate table drg'; +BEGIN +PMN_DROPSQL('drop table encounter_w_type'); +END; +/ -insert into drg +BEGIN +PMN_DROPSQL('drop table encounter_w_pay'); +END; +/ + +create table drg as select * from (select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from (select patient_num,encounter_num,drg_type,max(drg) drg from @@ -70,74 +41,117 @@ select * from inner join demographic d on f.patient_num=d.patid inner join pcornet_enc enc on enc.c_basecode = f.concept_cd and enc.c_fullname like '\PCORI\ENCOUNTER\DRG\%') drg1 group by patient_num,encounter_num,drg_type) drg) drg -where rn=1; - -execute immediate 'create index drg_idx on drg (patient_num, encounter_num)'; ---GATHER_TABLE_STATS('drg'); - -insert into encounter(PATID - , ENCOUNTERID - , admit_date - , ADMIT_TIME - , DISCHARGE_DATE - , DISCHARGE_TIME - , PROVIDERID - , FACILITY_LOCATION - , ENC_TYPE - , FACILITYID - , DISCHARGE_DISPOSITION - , DISCHARGE_STATUS - , DRG - , DRG_TYPE - , ADMITTING_SOURCE - , PAYER_TYPE_PRIMARY - , RAW_PAYER_TYPE_PRIMARY - , RAW_PAYER_NAME_PRIMARY - , RAW_PAYER_ID_PRIMARY) - -select distinct v.patient_num, - v.encounter_num, - start_Date, - to_char(start_Date,'HH24:MI'), - end_Date, - to_char(end_Date,'HH24:MI'), - providerid, - 'NI' location_zip, /* See TODO above */ - (case when pcori_enctype is not null then pcori_enctype else 'UN' end) enc_type, - 'NI' facility_id, /* See TODO above */ - CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_disposition END, - CASE WHEN pcori_enctype='AV' THEN 'NI' ELSE discharge_status END, - drg.drg, - drg_type, - CASE WHEN admitting_source IS NULL THEN 'NI' ELSE admitting_source END admitting_source, - null, - null, - null, - null -from i2b2visit v inner join demographic d on v.patient_num=d.patid -left outer join drg -- This section is bugfixed to only include 1 drg if multiple DRG types exist in a single encounter... - on drg.patient_num=v.patient_num and drg.encounter_num=v.encounter_num -left outer join --- Encounter type. Note that this requires a full table scan on the ontology table, so it is not particularly efficient. -(select patient_num, encounter_num, inout_cd,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,2) pcori_enctype from i2b2visit v - inner join pcornet_enc e on c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%') enctype - on enctype.patient_num=v.patient_num and enctype.encounter_num=v.encounter_num ---left outer join payer p on p.patient_num = v.patient_num and p.encounter_num = v.encounter_num -; - --- TODO : Re-enable indices!!!! -execute immediate 'create unique index encounter_pk on encounter (ENCOUNTERID)'; -execute immediate 'create index encounter_idx on encounter (PATID, ENCOUNTERID)'; -GATHER_TABLE_STATS('ENCOUNTER'); +where rn=1 +/ + +create index drg_idx on drg (patient_num, encounter_num) +/ + +create table encounter_key as +select patient_num PATID +, encounter_num ENCOUNTERID +, start_date ADMIT_DATE +, end_Date DISCHARGE_DATE +, providerid PROVIDERID +, admitting_source +, discharge_disposition RAW_DISCHARGE_DISPOSITION +, discharge_status RAW_DISCHARGE_STATUS +, inout_cd +from i2b2visit v +join demographic d on v.patient_num=d.patid +/ + +create table encounter_w_drg as +select en.* +, drg DRG +, drg_type DRG_TYPE +from encounter_key en +left join drg on drg.patient_num = en.patid and drg.encounter_num = en.encounterid +/ -end PCORNetEncounter; +create table encounter_w_type as +select en.* +, case when SUBSTR(pcori_basecode, INSTR(pcori_basecode, ':') + 1, 2) is not null then SUBSTR(pcori_basecode, INSTR(pcori_basecode, ':') + 1, 2) else 'UN' end ENC_TYPE +, case when SUBSTR(pcori_basecode, INSTR(pcori_basecode, ':') + 1, 2) = 'AV' then 'NI' else raw_discharge_disposition end DISCHARGE_DISPOSITION +, case when SUBSTR(pcori_basecode, INSTR(pcori_basecode, ':') + 1, 2) = 'AV' then 'NI' else raw_discharge_status end DISCHARGE_STATUS +from encounter_w_drg en +left join pcornet_enc e on e.c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%' / + +create table encounter_w_pay as +select en.* +, f.instance_num +, f.tval_char RAW_PAYER_NAME_PRIMARY +, f.concept_cd RAW_PAYER_ID_PRIMARY +from pcornet_cdm.encounter_w_type en +left join i2b2fact f on f.patient_num = en.patid and f.encounter_num = en.encounterid and f.concept_cd like 'O2|PAYER_PRIMARY:%' +-- IDX fails, as multiple payers are mapped to a single encounter_num. +-- TODO: determine the rollup logic used in heron to identify the primary encounter. +-- and (f.concept_cd like 'O2|PAYER_PRIMARY:%' or 'IDX|PAYER_PRIMARY:%') +/ + +create table encounter_w_fin as +select en.* +, pm.code PAYER_TYPE_PRIMARY +, sf.tval_char RAW_PAYER_TYPE_PRIMARY +from pcornet_cdm.encounter_w_pay en +left join blueherondata.supplemental_fact sf on en.instance_num = sf.instance_num +left join pcornet_cdm.payer_map pm on (pm.payer_name = en.raw_payer_name_primary and en.raw_payer_id_primary like 'O2|PAYER_PRIMARY:%' +and pm.financial_class = sf.tval_char) +-- IDX doesn't provide a financial class. Could be eliminated from the O2 mapping but it's actually more informative than +-- the payer name for deciding the payer type. +--or (pm.payer_name = en.raw_payer_name_primary and en.raw_payer_id_primary like 'IDX|PAYER_PRIMARY:%') +/ + +create table encounter as +select en.patid +, en.encounterid +, en.admit_date +, to_char(en.admit_date, 'HH24:MI') ADMIT_TIME +, en.discharge_date +, to_char(en.discharge_date,'HH24:MI') DISCHARGE_TIME +, en.providerid +, cast(null as varchar(3)) FACILITY_LOCATION +, en.enc_type +, cast(null as varchar(50)) FACILITYID +, en.discharge_disposition +, en.discharge_status +, en.drg +, en.drg_type +, en.admitting_source +, en.payer_type_primary +, cast(null as varchar(5)) PAYER_TYPE_SECONDARY +, cast(null as varchar(50)) FACILITY_TYPE +, cast(null as varchar(50)) RAW_SITEID +, cast(null as varchar(50)) RAW_ENC_TYPE +, cast(null as varchar(50)) RAW_DISCHARGE_DISPOSITION +, cast(null as varchar(50)) RAW_DISCHARGE_STATUS +, cast(null as varchar(50)) RAW_DRG_TYPE +, cast(null as varchar(50)) RAW_ADMITTING_SOURCE +, cast(null as varchar(50)) RAW_FACILITY_TYPE +, en.raw_payer_type_primary +, en.raw_payer_name_primary +, en.raw_payer_id_primary +, cast(null as varchar(50)) RAW_PAYER_TYPE_SECONDARY +, cast(null as varchar(50)) RAW_PAYER_NAME_SECONDARY +, cast(null as varchar(50)) RAW_PAYER_ID_SECONDARY +from encounter_w_fin en +/ + +create unique index encounter_pk on encounter (ENCOUNTERID) +/ + +create index encounter_idx on encounter (PATID, ENCOUNTERID) +/ + BEGIN -PCORNetEncounter(); +GATHER_TABLE_STATS('ENCOUNTER'); END; / + update cdm_status set end_time = sysdate, records = (select count(*) from encounter) where task = 'encounter' / + select records from cdm_status where task = 'encounter' \ No newline at end of file From 0357a4a42849abdfbacd893e21adc3058ef4abca Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 24 Jul 2018 15:45:50 -0500 Subject: [PATCH 442/507] Adding encounter_w_fin to drop statements. --- Oracle/encounter.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index eea2898..2434ed7 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -33,6 +33,11 @@ PMN_DROPSQL('drop table encounter_w_pay'); END; / +BEGIN +PMN_DROPSQL('drop table encounter_w_fin'); +END; +/ + create table drg as select * from (select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from From 9a18f1f0ecdc7549e3b98a01cffbd3f52863e058 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 26 Jul 2018 09:04:42 -0500 Subject: [PATCH 443/507] Minor syntax cleanup of encounter. --- Oracle/encounter.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 2434ed7..383ece8 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -76,7 +76,7 @@ left join drg on drg.patient_num = en.patid and drg.encounter_num = en.encounter create table encounter_w_type as select en.* -, case when SUBSTR(pcori_basecode, INSTR(pcori_basecode, ':') + 1, 2) is not null then SUBSTR(pcori_basecode, INSTR(pcori_basecode, ':') + 1, 2) else 'UN' end ENC_TYPE +, nvl(SUBSTR(pcori_basecode, INSTR(pcori_basecode, ':') + 1, 2), 'UN') ENC_TYPE , case when SUBSTR(pcori_basecode, INSTR(pcori_basecode, ':') + 1, 2) = 'AV' then 'NI' else raw_discharge_disposition end DISCHARGE_DISPOSITION , case when SUBSTR(pcori_basecode, INSTR(pcori_basecode, ':') + 1, 2) = 'AV' then 'NI' else raw_discharge_status end DISCHARGE_STATUS from encounter_w_drg en @@ -88,7 +88,7 @@ select en.* , f.instance_num , f.tval_char RAW_PAYER_NAME_PRIMARY , f.concept_cd RAW_PAYER_ID_PRIMARY -from pcornet_cdm.encounter_w_type en +from encounter_w_type en left join i2b2fact f on f.patient_num = en.patid and f.encounter_num = en.encounterid and f.concept_cd like 'O2|PAYER_PRIMARY:%' -- IDX fails, as multiple payers are mapped to a single encounter_num. -- TODO: determine the rollup logic used in heron to identify the primary encounter. @@ -99,9 +99,9 @@ create table encounter_w_fin as select en.* , pm.code PAYER_TYPE_PRIMARY , sf.tval_char RAW_PAYER_TYPE_PRIMARY -from pcornet_cdm.encounter_w_pay en -left join blueherondata.supplemental_fact sf on en.instance_num = sf.instance_num -left join pcornet_cdm.payer_map pm on (pm.payer_name = en.raw_payer_name_primary and en.raw_payer_id_primary like 'O2|PAYER_PRIMARY:%' +from encounter_w_pay en +left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num +left join payer_map pm on (pm.payer_name = en.raw_payer_name_primary and en.raw_payer_id_primary like 'O2|PAYER_PRIMARY:%' and pm.financial_class = sf.tval_char) -- IDX doesn't provide a financial class. Could be eliminated from the O2 mapping but it's actually more informative than -- the payer name for deciding the payer type. From a63d28ea2d4b5ccd572d69af00f0f01048b0f97c Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 27 Jul 2018 15:56:02 -0500 Subject: [PATCH 444/507] Fix for all red line items in the CDM 4.1 EDC report. --- Oracle/encounter.sql | 14 +++++++------- Oracle/harvest.sql | 4 ++-- Oracle/lab_result_cm.sql | 14 ++++++++++---- Oracle/prescribing.sql | 20 ++++++++++++++++---- Oracle/pro_cm.sql | 2 +- curated_data/route_map.csv | 1 - 6 files changed, 36 insertions(+), 19 deletions(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 383ece8..736ab31 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -58,7 +58,7 @@ select patient_num PATID , start_date ADMIT_DATE , end_Date DISCHARGE_DATE , providerid PROVIDERID -, admitting_source +, admitting_source ADMITTING_SOURCE , discharge_disposition RAW_DISCHARGE_DISPOSITION , discharge_status RAW_DISCHARGE_STATUS , inout_cd @@ -109,8 +109,8 @@ and pm.financial_class = sf.tval_char) / create table encounter as -select en.patid -, en.encounterid +select cast(patid as varchar(50)) PATID +, cast(encounterid as varchar(50)) ENCOUNTERID , en.admit_date , to_char(en.admit_date, 'HH24:MI') ADMIT_TIME , en.discharge_date @@ -119,12 +119,12 @@ select en.patid , cast(null as varchar(3)) FACILITY_LOCATION , en.enc_type , cast(null as varchar(50)) FACILITYID -, en.discharge_disposition -, en.discharge_status +, cast(en.discharge_disposition as varchar(2)) DISCHARGE_DISPOSITION +, cast(en.discharge_status as varchar(2)) DISCHARGE_STATUS , en.drg , en.drg_type -, en.admitting_source -, en.payer_type_primary +, cast(en.admitting_source as varchar(2)) +, cast(en.payer_type_primary as varchar(5)) PAYER_TYPE_PRIMARY , cast(null as varchar(5)) PAYER_TYPE_SECONDARY , cast(null as varchar(50)) FACILITY_TYPE , cast(null as varchar(50)) RAW_SITEID diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index 9761395..9a41f74 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -33,7 +33,7 @@ CREATE TABLE harvest( REPORT_DATE_MGMT varchar(2) NULL, RESOLVE_DATE_MGMT varchar(2) NULL, PRO_DATE_MGMT varchar(2) NULL, - DEATH_DATE_MGTM varchar(2) NULL, + DEATH_DATE_MGMT varchar(2) NULL, MEDADMIN_START_DATE_MGMT varchar(2) NULL, MEDADMIN_STOP_DATE_MGMT varchar(2) NULL, OBSCLIN_DATE_MGMT varchar(2) NULL, @@ -66,7 +66,7 @@ execute immediate 'truncate table harvest'; INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, - ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, DEATH_DATE_MGTM, MEDADMIN_START_DATE_MGMT, MEDADMIN_STOP_DATE_MGMT, + ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, DEATH_DATE_MGMT, MEDADMIN_START_DATE_MGMT, MEDADMIN_STOP_DATE_MGMT, OBSCLIN_DATE_MGMT, OBSGEN_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 5a19c96..4c8f966 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -138,10 +138,10 @@ select distinct cast(lab.LAB_RESULT_CM_ID as varchar(19)) LAB_RESULT_CM_ID , cast(lab.PATID as varchar(50)) PATID , cast(lab.ENCOUNTERID as varchar(50)) ENCOUNTERID , lab.SPECIMEN_SOURCE -, nvl(lab.LAB_LOINC, 'NI') LAB_LOINC +, cast(nvl(lab.LAB_LOINC, 'NI') as varchar(10)) LAB_LOINC , 'NI' PRIORITY , 'NI' RESULT_LOC -, nvl(lab.LAB_LOINC, 'NI') LAB_PX +, cast(nvl(lab.LAB_LOINC, 'NI') as varchar(11)) LAB_PX , 'LC' LAB_PX_TYPE , lab.LAB_ORDER_DATE LAB_ORDER_DATE , lab.LAB_ORDER_DATE SPECIMEN_DATE @@ -158,14 +158,20 @@ select distinct cast(lab.LAB_RESULT_CM_ID as varchar(19)) LAB_RESULT_CM_ID when length(lab.RESULT_UNIT) > 11 then substr(lab.RESULT_UNIT, 1, 11) else trim(replace(upper(lab.RESULT_UNIT), '(CALC)', '')) end RESULT_UNIT -, lab.NORM_RANGE_LOW +, cast(lab.NORM_RANGE_LOW as varchar(10)) NORM_RANGE_LOW , case when lab.NORM_RANGE_LOW is not null and lab.NORM_RANGE_HIGH is not null then 'EQ' when lab.NORM_RANGE_LOW is not null and lab.NORM_RANGE_HIGH is null then 'GE' when lab.NORM_RANGE_LOW is null and lab.NORM_RANGE_HIGH is not null then 'NO' else 'NI' end NORM_MODIFIER_LOW -, lab.NORM_RANGE_HIGH +, cast(lab.NORM_RANGE_HIGH as varchar(10)) NORM_RANGE_HIGH +, case + when lab.NORM_RANGE_LOW is not null and lab.NORM_RANGE_HIGH is not null then 'EQ' + when lab.NORM_RANGE_LOW is not null and lab.NORM_RANGE_HIGH is null then 'NO' + when lab.NORM_RANGE_LOW is null and lab.NORM_RANGE_HIGH is not null then 'LE' + else 'NI' + end NORM_MODIFIER_HIGH , case nvl(nullif(lab.ABN_IND, ''), 'NI') when 'H' then 'AH' when 'L' then 'AL' when 'A' then 'AB' else 'NI' end ABN_IND , cast(null as varchar(50)) RAW_LAB_NAME , cast(null as varchar(50)) RAW_LAB_CODE diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 3369078..9154c78 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -149,7 +149,13 @@ left join create table prescribing_w_freq as select rx.* -, substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) rx_frequency +-- '09' as PRN has been eliminated from the list of valid frequencies and replaced with a PRN flag. +-- For backward compatibility and simplified mapping, '09' has been retained in frequency mapping +-- (see heron:med_freq_mod_map.csv) and overridden here. +, case + when substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) = '09' then 'OT' + else substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) + end rx_frequency from prescribing_w_refills rx left join (select instance_num @@ -210,12 +216,18 @@ left join create table prescribing_w_prn as select rx.* -, nvl(prn.tval_char, 'NO') rx_prn_flag +-- PRN is determined by a specific source system fact or a frequency of '09'. +-- Note that '09' is now mapped to a frequency of other. See comments in +-- 'create table prescribing_w_freq as' for related logic. +, case + when prn.tval_char = 'Y' or rx.rx_frequency = '09' then 'Y' + else 'N' + end rx_prn_flag from prescribing_w_basis rx left join (select instance_num , concept_cd - , 'YES' as tval_char + , 'Y' as tval_char from &&i2b2_data_schema.observation_fact where modifier_cd = 'MedObs:PRN' /* aka: @@ -240,7 +252,7 @@ left join route_map rm on lower(rt.tval_char) = lower(rm.route_name) create table prescribing_w_daw as select rx.* -, nvl(tval_char, 'NI') rx_dispense_as_written +, cast(nvl(tval_char, 'NI') as varchar(2)) rx_dispense_as_written from prescribing_w_route rx left join (select instance_num diff --git a/Oracle/pro_cm.sql b/Oracle/pro_cm.sql index 1cab3d5..4bef352 100644 --- a/Oracle/pro_cm.sql +++ b/Oracle/pro_cm.sql @@ -23,7 +23,7 @@ CREATE TABLE pro_cm( PRO_ITEM_VERSION varchar(50) NULL, PRO_MEASURE_NAME varchar(50) NULL, PRO_MEASURE_SEQ varchar(50) NULL, - PRO_MEASURE_SCORE varchar(50) NULL, + PRO_MEASURE_SCORE number(8) NULL, PRO_MEASURE_THETA number(8) NULL, PRO_MEASURE_SCALED_TSCORE number(8) NULL, PRO_MEASURE_STANDARD_ERROR number(8) NULL, diff --git a/curated_data/route_map.csv b/curated_data/route_map.csv index 0100508..60a6b8a 100644 --- a/curated_data/route_map.csv +++ b/curated_data/route_map.csv @@ -136,7 +136,6 @@ ORAL/RECTAL/IV,OT Osteochondrial,OT Otic,OTIC PEG Tube,PERCUTANEOUS -PEG Tube,PERCUTANEOUS PEG-J tube,PERCUTANEOUS Per Corpak Tube,OT Per Dobhoff Tube,NASOGASTRIC From bcf826ef472e1fbf9947c543e09738a60c4ff654 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 27 Jul 2018 16:08:40 -0500 Subject: [PATCH 445/507] Fixing discrepancies in facility highlighted in EDC report. --- Oracle/encounter.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 736ab31..0f0c300 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -116,9 +116,9 @@ select cast(patid as varchar(50)) PATID , en.discharge_date , to_char(en.discharge_date,'HH24:MI') DISCHARGE_TIME , en.providerid -, cast(null as varchar(3)) FACILITY_LOCATION +, cast('NI' as varchar(3)) FACILITY_LOCATION , en.enc_type -, cast(null as varchar(50)) FACILITYID +, cast('NI' as varchar(50)) FACILITYID , cast(en.discharge_disposition as varchar(2)) DISCHARGE_DISPOSITION , cast(en.discharge_status as varchar(2)) DISCHARGE_STATUS , en.drg @@ -126,7 +126,7 @@ select cast(patid as varchar(50)) PATID , cast(en.admitting_source as varchar(2)) , cast(en.payer_type_primary as varchar(5)) PAYER_TYPE_PRIMARY , cast(null as varchar(5)) PAYER_TYPE_SECONDARY -, cast(null as varchar(50)) FACILITY_TYPE +, cast('NI' as varchar(50)) FACILITY_TYPE , cast(null as varchar(50)) RAW_SITEID , cast(null as varchar(50)) RAW_ENC_TYPE , cast(null as varchar(50)) RAW_DISCHARGE_DISPOSITION From 60893d042747573c4243414b60d45aaf52d30692 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 27 Jul 2018 16:10:28 -0500 Subject: [PATCH 446/507] Adding column alias for admitting source. --- Oracle/encounter.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 0f0c300..2f7471b 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -123,7 +123,7 @@ select cast(patid as varchar(50)) PATID , cast(en.discharge_status as varchar(2)) DISCHARGE_STATUS , en.drg , en.drg_type -, cast(en.admitting_source as varchar(2)) +, cast(en.admitting_source as varchar(2)) ADMITTING_SOURCE , cast(en.payer_type_primary as varchar(5)) PAYER_TYPE_PRIMARY , cast(null as varchar(5)) PAYER_TYPE_SECONDARY , cast('NI' as varchar(50)) FACILITY_TYPE From acf89ff6ebd6e52c9d7e3ea145e28e6149ee323a Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 30 Jul 2018 09:49:08 -0500 Subject: [PATCH 447/507] Fixing join on unit map to recover lost prescrbing data. --- Oracle/prescribing.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/prescribing.sql b/Oracle/prescribing.sql index 9154c78..f62f817 100644 --- a/Oracle/prescribing.sql +++ b/Oracle/prescribing.sql @@ -94,7 +94,7 @@ select cast(prescribing_seq.nextval as varchar(19)) prescribingid , units_cd raw_rx_dose_ordered_unit from &&i2b2_data_schema.observation_fact rx join encounter en on rx.encounter_num = en.encounterid -join unit_map um on rx.units_cd = um.unit_name +left join unit_map um on rx.units_cd = um.unit_name where rx.modifier_cd in ('MedObs:Inpatient', 'MedObs:Outpatient') / From b34984343e44eb0c1e22799aa63d8ab843b590fb Mon Sep 17 00:00:00 2001 From: catechol Date: Mon, 30 Jul 2018 14:24:36 -0500 Subject: [PATCH 448/507] Adding gather stats for supplemental_fact to init. This could be moved back to cdm_prep in Heron code. Not sure what's the best fit. --- Oracle/pcornet_init.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index edbd2fa..6662670 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -17,6 +17,12 @@ select case when qty = 0 then 1/0 else 1 end inout_cd_populated from ( select rxcui from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id where 1=0 / +BEGIN +dbms_stats.gather_table_stats( OWNNAME => '"BLUEHERONDATA"', TABNAME => '"SUPPLEMENTAL_FACT"', +ESTIMATE_PERCENT => DBMS_STATS.AUTO_SAMPLE_SIZE, DEGREE => 16, CASCADE => TRUE ); +END; +/ + create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS BEGIN DBMS_STATS.GATHER_TABLE_STATS ( From 260431db8d6833efad79da5acbe21d00b444be64 Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 31 Jul 2018 15:06:19 -0500 Subject: [PATCH 449/507] Fix for orphaned encounterids in med_admin table. --- Oracle/med_admin.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 7db509a..92771f3 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -70,7 +70,8 @@ insert into med_admin(patid , raw_medadmin_route) with med_start as ( select patient_num, encounter_num, provider_id, start_date, end_date, concept_cd, modifier_cd, instance_num - from &&i2b2_data_schema.observation_fact + from &&i2b2_data_schema.observation_fact f + join encounter enc on enc.patid = f.patient_num and enc.encounterid = f.encounter_num where modifier_cd = 'MedObs|MAR:New Bag' or modifier_cd = 'MedObs|MAR:Downtime Given - New Bag' or modifier_cd = 'MedObs|MAR:Given - Without Order' @@ -96,7 +97,7 @@ with med_start as ( select med_start.patient_num , med_start.encounter_num , null - , med_start.provider_id + , null , med_start.start_date , to_char(med_start.start_date, 'HH24:MI') , med_start.end_date From 018e6b77ad87c8c125269661b64bcc8752e56936 Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 31 Jul 2018 15:43:49 -0500 Subject: [PATCH 450/507] Hack job to restore blood pressure facts to vitals For a proper fix, the pcornet_vital.concept_cd provided by scilhs-ontology will need to be updated. --- Oracle/vital.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/vital.sql b/Oracle/vital.sql index 06271cc..5c07c0d 100644 --- a/Oracle/vital.sql +++ b/Oracle/vital.sql @@ -113,7 +113,7 @@ from ( select '\PCORI\VITAL\TOBACCO\' concept_path FROM DUAL ) bp, pcornet_vital pm where pm.c_fullname like bp.concept_path || '%' - ) codes on codes.concept_cd = obs.concept_cd + ) codes on codes.concept_cd = obs.concept_cd or obs.concept_cd = 'KUH|FLO_MEAS_ID:5_DIASTOLIC' ) vit on vit.patid = pd.patid join encounter enc on enc.encounterid = vit.encounterid ) x From c37324f33c8e6378dc50abda7056c5988c4d7b46 Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 31 Jul 2018 17:06:10 -0500 Subject: [PATCH 451/507] Rollback of hack, solved in H2P and SCILHS instead. Level of effort proved to be minimal despite novel work in H2P and SCHILS. Tested okay in STAGEDEV. --- Oracle/vital.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/vital.sql b/Oracle/vital.sql index 5c07c0d..06271cc 100644 --- a/Oracle/vital.sql +++ b/Oracle/vital.sql @@ -113,7 +113,7 @@ from ( select '\PCORI\VITAL\TOBACCO\' concept_path FROM DUAL ) bp, pcornet_vital pm where pm.c_fullname like bp.concept_path || '%' - ) codes on codes.concept_cd = obs.concept_cd or obs.concept_cd = 'KUH|FLO_MEAS_ID:5_DIASTOLIC' + ) codes on codes.concept_cd = obs.concept_cd ) vit on vit.patid = pd.patid join encounter enc on enc.encounterid = vit.encounterid ) x From d792cf05f0f10b2b929249ad553ebd6721247365 Mon Sep 17 00:00:00 2001 From: catechol Date: Tue, 31 Jul 2018 17:35:58 -0500 Subject: [PATCH 452/507] Adding encounter dependency to med_admin. The dependency is needed for a join on encounter to eliminate med_admin facts outside the timeline required for CDM. --- i2p_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index 43acc9d..3c4485b 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -119,7 +119,7 @@ class med_admin(I2PScriptTask): script = Script.med_admin def requires(self) -> List[luigi.Task]: - return [pcornet_init(), loadRouteMap(), loadUnitMap()] + return [encounter(), loadRouteMap(), loadUnitMap()] class obs_clin(I2PScriptTask): From e50d28fbf354f65375b53c2789aa2617e8aa5066 Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 1 Aug 2018 15:28:05 -0500 Subject: [PATCH 453/507] 11th hour fixes for multiple payers mapping to the same encounter. --- Oracle/encounter.sql | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 2f7471b..adc13c9 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -83,29 +83,40 @@ from encounter_w_drg en left join pcornet_enc e on e.c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%' / +-- pay_order was added as a quick fix for the problem of multiple payers mapping to the same encounterid. +-- The partition / order by operation selects the primary payer by alphabetical order (not good). +-- Other options considered were precedence by encounter type, by financial class or by CDM code but +-- none proved clear and expedient. +-- TODO: rework payer selection to align with encounter rollup logic or determine a clear precedence from other criteria. create table encounter_w_pay as select en.* -, f.instance_num -, f.tval_char RAW_PAYER_NAME_PRIMARY -, f.concept_cd RAW_PAYER_ID_PRIMARY +, pay.instance_num +, pay.tval_char RAW_PAYER_NAME_PRIMARY +, pay.concept_cd RAW_PAYER_ID_PRIMARY from encounter_w_type en -left join i2b2fact f on f.patient_num = en.patid and f.encounter_num = en.encounterid and f.concept_cd like 'O2|PAYER_PRIMARY:%' --- IDX fails, as multiple payers are mapped to a single encounter_num. --- TODO: determine the rollup logic used in heron to identify the primary encounter. --- and (f.concept_cd like 'O2|PAYER_PRIMARY:%' or 'IDX|PAYER_PRIMARY:%') -/ - -create table encounter_w_fin as +left join + (select encounter_num + , patient_num + , instance_num + , concept_cd + , tval_char + , row_number() over (partition by patient_num, encounter_num order by tval_char) as pay_order + from i2b2fact + where concept_cd like 'O2|PAYER_PRIMARY:%' or concept_cd like 'IDX|PAYER_PRIMARY:%' + ) pay on en.patid = pay.patient_num and en.encounterid = pay.encounter_num and pay_order = 1 +/ + +-- IDX does not provide a financial class, hence the abbreviated condition. However, the payer type code can be +-- inferred from the payer name, particularly were IDX names and O2 names coincide. This might challenge the +-- do not impute rule specified in the CDM spec, but doesn't seem unreasonable. select en.* , pm.code PAYER_TYPE_PRIMARY , sf.tval_char RAW_PAYER_TYPE_PRIMARY from encounter_w_pay en -left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num -left join payer_map pm on (pm.payer_name = en.raw_payer_name_primary and en.raw_payer_id_primary like 'O2|PAYER_PRIMARY:%' -and pm.financial_class = sf.tval_char) --- IDX doesn't provide a financial class. Could be eliminated from the O2 mapping but it's actually more informative than --- the payer name for deciding the payer type. ---or (pm.payer_name = en.raw_payer_name_primary and en.raw_payer_id_primary like 'IDX|PAYER_PRIMARY:%') +left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num and sf.source_column = 'FINANCIAL_CLASS' +left join payer_map pm on pm.payer_name = en.raw_payer_name_primary +and (en.raw_payer_id_primary like 'O2|PAYER_PRIMARY:%' and pm.financial_class = sf.tval_char) +or (en.raw_payer_id_primary like 'IDX|PAYER_PRIMARY:%') / create table encounter as From e043324f27b7852c990796de658f7c75503b3dfa Mon Sep 17 00:00:00 2001 From: catechol Date: Wed, 1 Aug 2018 16:04:24 -0500 Subject: [PATCH 454/507] Restoring lost create table statement. Arrrrgggggh. --- Oracle/encounter.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index adc13c9..5c4c677 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -109,6 +109,7 @@ left join -- IDX does not provide a financial class, hence the abbreviated condition. However, the payer type code can be -- inferred from the payer name, particularly were IDX names and O2 names coincide. This might challenge the -- do not impute rule specified in the CDM spec, but doesn't seem unreasonable. +create table encounter_w_fin as select en.* , pm.code PAYER_TYPE_PRIMARY , sf.tval_char RAW_PAYER_TYPE_PRIMARY From 7814922bb07ce1d69a8ef9e2a3f67c5d0db98cb6 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 2 Aug 2018 17:55:37 -0500 Subject: [PATCH 455/507] Optimized med_admin by eliminating encounter join. --- Oracle/med_admin.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 92771f3..88eea49 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -71,8 +71,7 @@ insert into med_admin(patid with med_start as ( select patient_num, encounter_num, provider_id, start_date, end_date, concept_cd, modifier_cd, instance_num from &&i2b2_data_schema.observation_fact f - join encounter enc on enc.patid = f.patient_num and enc.encounterid = f.encounter_num - where modifier_cd = 'MedObs|MAR:New Bag' + where (modifier_cd = 'MedObs|MAR:New Bag' or modifier_cd = 'MedObs|MAR:Downtime Given - New Bag' or modifier_cd = 'MedObs|MAR:Given - Without Order' or modifier_cd = 'MedObs|MAR:Downtime Given' @@ -92,7 +91,8 @@ with med_start as ( or modifier_cd = 'MedObs|MAR:See OR/Proc Flowsheet' or modifier_cd = 'MedObs|MAR:Patch Applied' or modifier_cd = 'MedObs|MAR:Bolus from Syringe' - or modifier_cd = 'MedObs|MAR:Bolus.' + or modifier_cd = 'MedObs|MAR:Bolus.') + and encounter_num in (select encounterid from encounter) ) select med_start.patient_num , med_start.encounter_num @@ -125,6 +125,9 @@ or med_dose.modifier_cd = 'MedObs:MAR_Dose|tab' or med_dose.modifier_cd = 'MedObs:MAR_Dose|units' or med_dose.modifier_cd = 'MedObs:MAR_Dose|l' or med_dose.modifier_cd = 'MedObs:MAR_Dose|mg' +-- MedObs:Dose modifiers are for days supply, not discrete dose. Early designs used these values +-- but were dropped to comply with the CDM specs do not impute rule. Revisit this decision for +-- future CDM builds and restore or drop permanently. --or med_dose.modifier_cd = 'MedObs:Dose|puff' --or med_dose.modifier_cd = 'MedObs:Dose|drop' --or med_dose.modifier_cd = 'MedObs:Dose|cap' From ec101837542d6667b2cba154e9e514a24d412046 Mon Sep 17 00:00:00 2001 From: catechol Date: Thu, 2 Aug 2018 18:25:29 -0500 Subject: [PATCH 456/507] Updating cdm version in harvest insert. --- Oracle/harvest.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index 9a41f74..85cabc4 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -71,7 +71,7 @@ INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE, REFRESH_MED_ADMIN_DATE, REFRESH_OBS_CLIN_DATE, REFRESH_PROVIDER_DATE, REFRESH_OBS_GEN_DATE) - select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, + select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 4.1, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.PX_DATE_MGMT, hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, From 12e3a0064271f973262685936501b1f4a7c45e30 Mon Sep 17 00:00:00 2001 From: catechol Date: Fri, 3 Aug 2018 10:42:10 -0500 Subject: [PATCH 457/507] Removing trigger from provider build (botched provid on insert). --- Oracle/provider.sql | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/Oracle/provider.sql b/Oracle/provider.sql index 783bd37..a718dad 100644 --- a/Oracle/provider.sql +++ b/Oracle/provider.sql @@ -18,22 +18,6 @@ CREATE TABLE provider( ) / -BEGIN -PMN_DROPSQL('DROP sequence provider_seq'); -END; -/ - -create sequence provider_seq cache 2000 -/ - -create or replace trigger provider_trg -before insert on provider -for each row -begin - select provider_seq.nextval into :new.PROVIDERID from dual; -end; -/ - create or replace procedure PCORNetProvider as begin From 8f1340a8fe039603e84489314ba46d7e758256e7 Mon Sep 17 00:00:00 2001 From: schandaka Date: Tue, 9 Oct 2018 12:50:04 -0500 Subject: [PATCH 458/507] Update encounter.sql Fixed incorrect join syntax --- Oracle/encounter.sql | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 5c4c677..b4b991d 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -115,9 +115,8 @@ select en.* , sf.tval_char RAW_PAYER_TYPE_PRIMARY from encounter_w_pay en left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num and sf.source_column = 'FINANCIAL_CLASS' -left join payer_map pm on pm.payer_name = en.raw_payer_name_primary -and (en.raw_payer_id_primary like 'O2|PAYER_PRIMARY:%' and pm.financial_class = sf.tval_char) -or (en.raw_payer_id_primary like 'IDX|PAYER_PRIMARY:%') +left join payer_map pm on pm.payer_name = en.raw_payer_name_primary and (en.raw_payer_id_primary like 'O2|PAYER_PRIMARY:%' and pm.financial_class = sf.tval_char) + or( pm.payer_name = en.raw_payer_name_primary and (en.raw_payer_id_primary like 'IDX|PAYER_PRIMARY:%')) / create table encounter as @@ -171,4 +170,4 @@ set end_time = sysdate, records = (select count(*) from encounter) where task = 'encounter' / -select records from cdm_status where task = 'encounter' \ No newline at end of file +select records from cdm_status where task = 'encounter' From c821477eff9851965cd7af229eaefb8599c5b23e Mon Sep 17 00:00:00 2001 From: schandaka Date: Tue, 9 Oct 2018 15:55:55 -0500 Subject: [PATCH 459/507] Fixed the medicaid out of state duplicate row --- curated_data/payer_map.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/curated_data/payer_map.csv b/curated_data/payer_map.csv index 0273654..3abaf7d 100644 --- a/curated_data/payer_map.csv +++ b/curated_data/payer_map.csv @@ -222,7 +222,7 @@ MEDICAID KS HOM(102 REPLACEMEN,MEDICAID KS,2,MEDICAID UKH SP KS MEDICAID,MEDICAID KS,2,MEDICAID MEDICAID MO,MEDICAID MO,2,MEDICAID MEDICAID MO HMO,MEDICAID MO,2,MEDICAID -MEDICAID OUT OF STATE,MEDICAID OUT OF STATE,25,Medicaid - Out of State +MEDICAID OUT OF STATE,Medicaid OOS,25,Medicaid - Out of State KS MEDICAID PENDING,MEDICAID PENDING,2,MEDICAID MO MEDICAID PENDING,MEDICAID PENDING,2,MEDICAID MEDICAID,Medicaid,2,MEDICAID From 1c705a57df878ede87be31e8c840bcf72aeb177c Mon Sep 17 00:00:00 2001 From: schandaka Date: Tue, 9 Oct 2018 16:07:18 -0500 Subject: [PATCH 460/507] Removed duplicate entries for 'AGENCY' --- curated_data/payer_map.csv | 1 - 1 file changed, 1 deletion(-) diff --git a/curated_data/payer_map.csv b/curated_data/payer_map.csv index 3abaf7d..76464ea 100644 --- a/curated_data/payer_map.csv +++ b/curated_data/payer_map.csv @@ -3,7 +3,6 @@ AETNA USHC HMO/POS A,AETNA,5,PRIVATE HEALTH INSURANCE AETNA USHC HMO/POS B,AETNA,5,PRIVATE HEALTH INSURANCE AETNA USHC PPO/TRAD A,AETNA,5,PRIVATE HEALTH INSURANCE AETNA USHC PPO/TRAD B,AETNA,5,PRIVATE HEALTH INSURANCE -AGENCY,AGENCY,5,PRIVATE HEALTH INSURANCE BLUE CROSS BLUE SHIELD,AGENCY,6,BLUE CROSS/BLUE SHIELD CRIME VICTIMS KS,AGENCY,349,Other CRIMES VICTIMS MO,AGENCY,349,Other From 56d17fa8b8072a4138a92e27b7a31f4723460eb9 Mon Sep 17 00:00:00 2001 From: schandaka Date: Wed, 10 Oct 2018 09:09:28 -0500 Subject: [PATCH 461/507] Removed duplicated Medicaid OOS --- curated_data/payer_map.csv | 1 - 1 file changed, 1 deletion(-) diff --git a/curated_data/payer_map.csv b/curated_data/payer_map.csv index 76464ea..42eefbe 100644 --- a/curated_data/payer_map.csv +++ b/curated_data/payer_map.csv @@ -226,7 +226,6 @@ KS MEDICAID PENDING,MEDICAID PENDING,2,MEDICAID MO MEDICAID PENDING,MEDICAID PENDING,2,MEDICAID MEDICAID,Medicaid,2,MEDICAID ALT MO MEDICAID,Medicaid OOS,25,Medicaid - Out of State -MEDICAID OUT OF STATE,Medicaid OOS,25,Medicaid - Out of State MO MEDICAID,Medicaid OOS,25,Medicaid - Out of State UKH SP OOS MEDICAID,Medicaid OOS,25,Medicaid - Out of State ALT CENPATICO KS MCAID BEHAVIORAL,Medicaid Repl KS,2,MEDICAID From b50b8462cb671d367eaae3284448547b9d7ab885 Mon Sep 17 00:00:00 2001 From: schandaka Date: Tue, 27 Nov 2018 10:21:00 -0600 Subject: [PATCH 462/507] Fixed missing egfr loinc codes issue --- Oracle/lab_result_cm.sql | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 4c8f966..055a76a 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -190,10 +190,18 @@ BEGIN GATHER_TABLE_STATS('LAB_RESULT_CM'); END; / +update pcornet_cdm.lab_result_cm +set lab_loinc='48642-3' +where raw_facility_code like '%KUH|COMPONENT_ID:191' + +update pcornet_cdm.lab_result_cm +set lab_loinc='48643-1' +where raw_facility_code like '%KUH|COMPONENT_ID:200' + update cdm_status set end_time = sysdate, records = (select count(*) from lab_result_cm) where task = 'lab_result_cm' / -select records from cdm_status where task = 'lab_result_cm' \ No newline at end of file +select records from cdm_status where task = 'lab_result_cm' From 5d6a3e4aa76e9fbd049bbccab8159f31e425cfe2 Mon Sep 17 00:00:00 2001 From: schandaka Date: Tue, 27 Nov 2018 10:52:48 -0600 Subject: [PATCH 463/507] Added code to load the result unit manual curation csv --- i2p_tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/i2p_tasks.py b/i2p_tasks.py index 3c4485b..0d648fe 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -285,6 +285,13 @@ class loadLabNormal(LoadCSV): def requires(self) -> List[luigi.Task]: return [pcornet_init()] + + class loadLabRUnit(LoadCSV): + taskName = 'LABRUNIT' + csvname = 'curated_data/resultunit_manualcuration.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] class loadHarvestLocal(LoadCSV): From 5486ebd29f93b407b5c22b0143bb689a99a1938e Mon Sep 17 00:00:00 2001 From: schandaka Date: Tue, 27 Nov 2018 10:54:55 -0600 Subject: [PATCH 464/507] Adding resultunit manualcuration file --- curated_data/resultunit_manualcuration.csv | 162 +++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 curated_data/resultunit_manualcuration.csv diff --git a/curated_data/resultunit_manualcuration.csv b/curated_data/resultunit_manualcuration.csv new file mode 100644 index 0000000..03a899c --- /dev/null +++ b/curated_data/resultunit_manualcuration.csv @@ -0,0 +1,162 @@ +FREQ,RESULT_UNIT,UCUM_CODE,UCUM_DESCRIPTIVE_TEXT +1123,#,OT,Other +2221154,/HPF,OT,Other +46026,/LPF,OT,Other +80338,/UL,OT,Other +18887,[IU]/ML,[IU]/mL,international unit per milliliter +10151,10*3/UL,10*3/uL,thousand per microliter +2140,10*6/UL,10*6/uL,million per microliter +2419,10^3,OT,Other +6451,10^3/UL,10*3/uL,thousand per microliter +1992,1000/UL,10*3/uL,thousand per microliter +10563,10E9/L,OT,Other +11642,AI,OT,Other +3141,AU,OT,Other +174064,BPM,OT,Other +2842,CC,OT,Other +689040,CELLS/UL,OT,Other +1366532,CM,OT,Other +5972,CM (<=0.5),OT,Other +68355,CM (<=4.4),OT,Other +73132,CM (<=5.3),OT,Other +5161,cm (2.0-2.8,OT,Other +77157,cm (2.0-3.5,OT,Other +77545,cm (2.4-4.2,OT,Other +5148,cm (2.7-3.3,OT,Other +76747,cm (5.6-8.6,OT,Other +5149,cm (7.1-7.9,OT,Other +74761,CM2,OT,Other +83688,CM2 (<=18),OT,Other +26975,CM3,OT,Other +1010,CMM,OT,Other +14348,COPIES/ML,OT,Other +7561,DEG,OT,Other +3020,E10 CELLS,OT,Other +3018,E6 CELLS,OT,Other +3018,E6 CELLS/KG,OT,Other +6314023,FL,fL,femtoliter +7565,G,g,gram +1577,G/24 H,g/(24.h),gram per 24 hour +8288964,G/DL,g/dL,gram per deciliter +7412,G/M2,OT,Other +3149751,GM/DL,g/dL,gram per deciliter +1153,GMS,OT,Other +1122,GPL,g/L,gram per liter +8459,GPL/ML,OT,Other +1678,H,h,hour +1678,H,H,Henry +11111,IN,OT,Other +7319,INDEX,OT,Other +2123,INDEX VALUE,OT,Other +1756,INR,OT,Other +20221,IU/L,OT,Other +120014,IU/ML,OT,Other +3099,K/CMM,OT,Other +4645,K/CUMM,OT,Other +1376,K/MM3,OT,Other +15274855,K/UL,OT,Other +4353,KG,kg,kilogram +39896,KU/L,OT,Other +11408,LBS,OT,Other +1210,LOG IU/ML,OT,Other +1325,M,OT,Other +1118508,M/S,OT,Other +3045285,M/UL,mU/L,milli enzyme unit per liter +169686,M2,m2,square meter +1183,MCG/24 H,OT,Other +280255,MCG/DL,OT,Other +4738,mcg/dL (cal,OT,Other +2570,MCG/KG/MIN,OT,Other +9586,MCG/L,OT,Other +16459,mcg/mg crea,OT,Other +162148,MCG/ML,OT,Other +101181,MCI,OT,Other +21359,MCMOL/L,OT,Other +311906,MCU/ML,OT,Other +39352,MEG/L,OT,Other +150846,MEQ/L,OT,Other +2984,METER,OT,Other +29104,METS,OT,Other +1192,MG/24 H,mg/(24.h),milligram per 24 hour +15652,MG/24 HRS,mg/(24.h),milligram per 24 hour +2007,MG/24HR,mg/(24.h),milligram per 24 hour +24352219,MG/DL,mg/dL,milligram per deciliter +267299,mg/dL (calc,mg/dL,milligram per deciliter +2975,MG/G CREAT,mg/g{creat},milligram per gram of creatinine +8998,MG/GRAM,mg/g,milligram per gram +24634,MG/L,mg/L,milligram per liter +3017,MIL/UL,m[IU]/mL,milli international unit per milliliter +167079,MILLION/UL,OT,Other +61834,MIN,min,minute +116367,MIU/L,m[IU]/L,milli international unit per liter +61043,MIU/ML,m[IU]/mL,milli international unit per milliliter +151959,ML,mL,milliliter +4050252,ML/MIN,mL/min,milliliter per minute +6362259,ML/MIN/1.73,OT,Other +460610,mL/min/1.73,OT,Other +1517,ml/min/1.73,OT,Other +17822,MLS,OT,Other +8022,MM,OT,Other +14015,MM/H,mm/h,millimeter per hour +98074,MM/HR,mm/h,millimeter per hour +1285125,MMHG,mm[Hg],millimeter of mercury +3923,MMOL/24 HRS,mmol/(24.h),millimole per 24 hour +14747546,MMOL/L,mmol/L,millimole per liter +20357,MOS/KG,mosm/kg,milliosmole per kilogram +23350,MOSMOL/KG,mosm/kg,milliosmole per kilogram +1536,MPH,OT,Other +1071,MPL,OT,Other +8108,MPL/ML,OT,Other +26039,MS,OT,Other +152378,MSEC,OT,Other +21140,MU/ML,mU/mL,milli enzyme unit per milliliter +148859,NG/DL,ng/dL,nanogram per deciliter +1181293,NG/ML,ng/mL,nanogram per millliiter +37283,NG/ML DDU,OT,Other +1658,NG/ML FEU,OT,Other +26640416,NI,NI,No information +1628,NMBCE/L,OT,Other +11552,NMOL/L,nmol/L,nanomole per liter +25756,NMOL/ML,nmol/mL,nanomole per milliliter +1123,NMOLES,OT,Other +19633313,PERCENT,%,percent +3233164,PG,pg,picogram +273580,PG/ML,pg/mL,picogram per milliliter +8363,RATIO,OT,Other +10252,RNA COPIES/,OT,Other +33807,S,S,Siemens +33807,S,s,second +505524,SEC,s,second +1184,SECONDS,s,second +5070,SECS,s,second +4564,SGU,OT,Other +4340,SMU,OT,Other +1647,T/CMM,OT,Other +1529,TH/CMM,OT,Other +5180,TH/UL,OT,Other +2212,THOU/UL,OT,Other +2239,THOUS,OT,Other +4073,THOUS/UL,OT,Other +335245,THOUSAND/UL,OT,Other +134897,TITER,OT,Other +7394762,U/L,U/L,enzyme unit per liter +90061,U/ML,U/mL,enzyme unit per milliliter +9425,UG/DL,ug/dL,microgram per deciliter +1636,UG/L,ug/L,microgram per liter +5618,UG/MG,OT,Other +1874,UG/ML,ug/mL,microgram per milliliter +34358,UIU/ML,u[IU]/mL,micro international unit per milliliter +39742,UL,U/L,enzyme unit per liter +28578,UMOL/L,umol/L,micromole per liter +5970,UNIT/L,U/L,enzyme unit per liter +2793,UNITS,OT,Other +4069,UNITS/L,U/L,enzyme unit per liter +2341,WEEKS,wk,week +1707,X10 3/UL,OT,Other +1205,X10.E3/UL,OT,Other +4231,X10^3,OT,Other +6873,X10^3/UL,OT,Other +2409,X1000,OT,Other +1333,X10-3/UL,OT,Other +15873,X10E3/UL,OT,Other +2272,X10E6/UL,OT,Other From 54bcf7d7282521b7bebdd798755189a216f13c31 Mon Sep 17 00:00:00 2001 From: schandaka Date: Tue, 27 Nov 2018 11:36:44 -0600 Subject: [PATCH 465/507] Update lab_result_cm.sql --- Oracle/lab_result_cm.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 055a76a..3f69b56 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -190,6 +190,9 @@ BEGIN GATHER_TABLE_STATS('LAB_RESULT_CM'); END; / +UPDATE pcornet_cdm.lab_result_cm lab +SET lab.result_unit = (SELECT mc.ucum_code FROM pcornet_cdm.resultunit_manualcuration mc WHERE lab.result_unit = mc.result_unit) + update pcornet_cdm.lab_result_cm set lab_loinc='48642-3' where raw_facility_code like '%KUH|COMPONENT_ID:191' From e50ce3865c27ca589b67d9ea59fe46091bc16ddf Mon Sep 17 00:00:00 2001 From: schandaka Date: Fri, 7 Dec 2018 13:21:27 -0600 Subject: [PATCH 466/507] Removed unnecessary indentation the file --- i2p_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index 0d648fe..96c9178 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -286,7 +286,7 @@ class loadLabNormal(LoadCSV): def requires(self) -> List[luigi.Task]: return [pcornet_init()] - class loadLabRUnit(LoadCSV): +class loadLabRUnit(LoadCSV): taskName = 'LABRUNIT' csvname = 'curated_data/resultunit_manualcuration.csv' From 63a3783a035564b74861d1cae191357c075f27b0 Mon Sep 17 00:00:00 2001 From: schandaka Date: Mon, 17 Dec 2018 13:41:33 -0600 Subject: [PATCH 467/507] SQL not properly ended --- Oracle/lab_result_cm.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index 3f69b56..d54ad34 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -191,16 +191,16 @@ GATHER_TABLE_STATS('LAB_RESULT_CM'); END; / UPDATE pcornet_cdm.lab_result_cm lab -SET lab.result_unit = (SELECT mc.ucum_code FROM pcornet_cdm.resultunit_manualcuration mc WHERE lab.result_unit = mc.result_unit) +SET lab.result_unit = (SELECT mc.ucum_code FROM pcornet_cdm.resultunit_manualcuration mc WHERE lab.result_unit = mc.result_unit); update pcornet_cdm.lab_result_cm set lab_loinc='48642-3' -where raw_facility_code like '%KUH|COMPONENT_ID:191' +where raw_facility_code like '%KUH|COMPONENT_ID:191'; update pcornet_cdm.lab_result_cm set lab_loinc='48643-1' -where raw_facility_code like '%KUH|COMPONENT_ID:200' +where raw_facility_code like '%KUH|COMPONENT_ID:200'; update cdm_status set end_time = sysdate, records = (select count(*) from lab_result_cm) From cbad55b9faee18c58385da5b2dd184edf30648f5 Mon Sep 17 00:00:00 2001 From: schandaka Date: Wed, 19 Dec 2018 16:09:16 -0600 Subject: [PATCH 468/507] SQL not ended properly --- Oracle/lab_result_cm.sql | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index d54ad34..bd0fc0f 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -190,18 +190,7 @@ BEGIN GATHER_TABLE_STATS('LAB_RESULT_CM'); END; / -UPDATE pcornet_cdm.lab_result_cm lab -SET lab.result_unit = (SELECT mc.ucum_code FROM pcornet_cdm.resultunit_manualcuration mc WHERE lab.result_unit = mc.result_unit); - -update pcornet_cdm.lab_result_cm -set lab_loinc='48642-3' -where raw_facility_code like '%KUH|COMPONENT_ID:191'; - - -update pcornet_cdm.lab_result_cm -set lab_loinc='48643-1' -where raw_facility_code like '%KUH|COMPONENT_ID:200'; - + update cdm_status set end_time = sysdate, records = (select count(*) from lab_result_cm) where task = 'lab_result_cm' From cce49c280520dd5618c30ef4603616eb4428a7dd Mon Sep 17 00:00:00 2001 From: schandaka Date: Sat, 16 Feb 2019 20:03:25 -0600 Subject: [PATCH 469/507] Made changes to get the schema names from the variables --- Oracle/heron_encounter_style.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index a06d2e4..17e74a2 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -347,8 +347,8 @@ with codes as ( replace(pe.pcori_basecode, 'ADMITTING_SOURCE:', '') pcori_code from pcornet_mapping pm - join blueherondata.concept_dimension cd on cd.concept_path like pm.local_path || '%' - join blueheronmetadata.pcornet_enc pe on pe.c_fullname = pm.pcori_path + join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' + join "&&i2b2_meta_data_schema".pcornet_enc pe on pe.c_fullname = pm.pcori_path where pm.pcori_path like '\PCORI\ENCOUNTER\ADMITTING_SOURCE\%' ) select @@ -356,7 +356,7 @@ select -- Prioritize status: Last OT, UN, NI decode(codes.pcori_code, 'AF',0,'AL',1,'AV',2,'ED',3,'HH',4,'HO',5,'HS',6,'IP',7, 'NH',8,'RH',9,'RS',10,'SN',11,'OT',12,'UN',13,'NI',14, 99) as_rank -from blueherondata.observation_fact obs +from "&&i2b2_data_schema".observation_fact obs join codes on codes.concept_cd = obs.concept_cd; create index admit_source_enc_rank_idx on admit_source_enc_code_rank(encounter_num, as_rank); From a715c677a4cf35a48c7391486c7b06a4e0d332f9 Mon Sep 17 00:00:00 2001 From: schandaka Date: Sun, 17 Feb 2019 07:16:34 -0600 Subject: [PATCH 470/507] Fixed incorrect variable name --- Oracle/heron_encounter_style.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql index 17e74a2..2db90d6 100644 --- a/Oracle/heron_encounter_style.sql +++ b/Oracle/heron_encounter_style.sql @@ -348,7 +348,7 @@ with codes as ( from pcornet_mapping pm join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' - join "&&i2b2_meta_data_schema".pcornet_enc pe on pe.c_fullname = pm.pcori_path + join "&&i2b2_meta_schema".pcornet_enc pe on pe.c_fullname = pm.pcori_path where pm.pcori_path like '\PCORI\ENCOUNTER\ADMITTING_SOURCE\%' ) select From 18b169e61cbe184aa2d28e3bdac991a9d76829e0 Mon Sep 17 00:00:00 2001 From: schandaka Date: Tue, 19 Feb 2019 09:22:00 -0600 Subject: [PATCH 471/507] Updated the schema name to the relevant string parameter --- Oracle/pcornet_init.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql index 6662670..13f2aab 100644 --- a/Oracle/pcornet_init.sql +++ b/Oracle/pcornet_init.sql @@ -18,7 +18,7 @@ select rxcui from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id where 1=0 / BEGIN -dbms_stats.gather_table_stats( OWNNAME => '"BLUEHERONDATA"', TABNAME => '"SUPPLEMENTAL_FACT"', +dbms_stats.gather_table_stats( OWNNAME => '"&&i2b2_data_schema"', TABNAME => '"SUPPLEMENTAL_FACT"', ESTIMATE_PERCENT => DBMS_STATS.AUTO_SAMPLE_SIZE, DEGREE => 16, CASCADE => TRUE ); END; / From 36777048d97d01b7fa50ae94804fcbec57af98dd Mon Sep 17 00:00:00 2001 From: schandaka Date: Mon, 11 Mar 2019 09:06:50 -0500 Subject: [PATCH 472/507] using the 'cdm' provider table to match with the deid provider table --- Oracle/provider.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/provider.sql b/Oracle/provider.sql index a718dad..ff25bce 100644 --- a/Oracle/provider.sql +++ b/Oracle/provider.sql @@ -35,7 +35,7 @@ select pd.provider_id , pd.provider_npi , case when pd.provider_npi is not null then 'Y' else 'N' end as provider_npi_flag , substr(sp.descriptive_text, 1, 50) - from &&i2b2_data_schema.provider_dimension pd + from &&i2b2_data_schema.provider_dimension_cdm pd left join provider_specialty_map sm on sm.npi = pd.provider_npi left join provider_specialty_code sp on sm.specialty = sp.code; From db7cf8809d810385d2e6d4e4f1e66ac7cb02ed57 Mon Sep 17 00:00:00 2001 From: schandaka Date: Wed, 13 Mar 2019 16:16:07 -0500 Subject: [PATCH 473/507] Removed specialty_map task --- i2p_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index 96c9178..e9720db 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -256,7 +256,7 @@ class provider(I2PScriptTask): script = Script.provider def requires(self) -> List[luigi.Task]: - return [loadSpecialtyMap(), loadSpecialtyCode(), encounter()] + return [loadSpecialtyCode(), encounter()] class vital(I2PScriptTask): From 16bbe845301919f6deb7378874c100d97b4afe7b Mon Sep 17 00:00:00 2001 From: schandaka Date: Thu, 14 Mar 2019 15:22:07 -0500 Subject: [PATCH 474/507] Changed the hard coded schema reference to 'BLUEHERONDATA' --- Oracle/med_admin.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 88eea49..3b97ada 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -137,7 +137,7 @@ or med_dose.modifier_cd = 'MedObs:MAR_Dose|mg' --or med_dose.modifier_cd = 'MedObs:Dose|tab' --or med_dose.modifier_cd = 'MedObs:Dose|mg' ) -left join BLUEHERONMETADATA.pcornet_med med_p on med_p.c_basecode = med_start.concept_cd +left join &&i2b2_data_schema.pcornet_med med_p on med_p.c_basecode = med_start.concept_cd left join unit_map um on um.unit_name = med_dose.units_cd ; From c3eafaff226d92cde39b3b659c82cb0e58a9abf9 Mon Sep 17 00:00:00 2001 From: schandaka Date: Thu, 14 Mar 2019 15:31:56 -0500 Subject: [PATCH 475/507] Removed the error table creation for condition --- Oracle/condition.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Oracle/condition.sql b/Oracle/condition.sql index b572662..8ca921e 100644 --- a/Oracle/condition.sql +++ b/Oracle/condition.sql @@ -73,7 +73,7 @@ insert into sourcefact2 execute immediate 'create index sourcefact2_idx on sourcefact2 (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('SOURCEFACT2'); -create_error_table('CONDITION'); +/*create_error_table('CONDITION');*/ insert into condition (patid, encounterid, report_date, resolve_date, condition, condition_type, condition_status, condition_source) select distinct factline.patient_num, min(factline.encounter_num) encounterid, min(factline.start_date) report_date, NVL(max(factline.end_date),null) resolve_date, diag.pcori_basecode, @@ -92,7 +92,7 @@ and factline.start_date=sf.start_Date where diag.c_fullname like '\PCORI\DIAGNOSIS\%' and sf.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\%' group by factline.patient_num, diag.pcori_basecode, diag.c_fullname -log errors into ERR$_CONDITION reject limit unlimited +/*log errors into ERR$_CONDITION reject limit unlimited*/ ; execute immediate 'create index condition_idx on condition (PATID, ENCOUNTERID)'; @@ -108,4 +108,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from condition) where task = 'condition' / -select records from cdm_status where task = 'condition' \ No newline at end of file +select records from cdm_status where task = 'condition' From 6c81ba3cf014a3d10f91e8a918f74e637509480a Mon Sep 17 00:00:00 2001 From: schandaka Date: Tue, 23 Apr 2019 12:44:20 -0500 Subject: [PATCH 476/507] Added code to remove lab outliers (convert mg/dL to gm/dL) --- Oracle/pcornet_loader.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Oracle/pcornet_loader.sql b/Oracle/pcornet_loader.sql index 2fb7b8f..33065cd 100644 --- a/Oracle/pcornet_loader.sql +++ b/Oracle/pcornet_loader.sql @@ -32,6 +32,11 @@ begin The CDM wants height in inches and weight in pounds. */ update vital v set v.ht = v.ht / 2.54; update vital v set v.wt = v.wt / 16; + + /* Result units used by KUH are mg/dL but the CDM spec requires a gm/dL*/ + update pcornet_cdm.lab_result_cm + set result_num=result_num/1000 + where lab_loinc in('2862-1','26474-7'); /* Remove rows from the PRESCRIBING table where RX_* fields are null TODO: Remove this when fixed in HERON @@ -60,4 +65,4 @@ update cdm_status set end_time = sysdate, records = 0 where task = 'pcornet_loader' / -select records + 1 from cdm_status where task = 'pcornet_loader' \ No newline at end of file +select records + 1 from cdm_status where task = 'pcornet_loader' From 9a755353072f9bc826cc0afb9cc46f9c0ddce272 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 31 Jul 2019 13:01:59 -0500 Subject: [PATCH 477/507] Added DX_DATE column to the diagnosis table --- Oracle/diagnosis.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 281123e..8e628c7 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -12,6 +12,7 @@ CREATE TABLE diagnosis( ENCOUNTERID varchar(50) NOT NULL, ENC_TYPE varchar(2) NULL, ADMIT_DATE date NULL, + DX_DATE date NULL, PROVIDERID varchar(50) NULL, DX varchar(18) NOT NULL, DX_TYPE varchar(2) NOT NULL, @@ -177,7 +178,7 @@ insert into poafact execute immediate 'create index poafact_idx on poafact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('POAFACT'); -insert into diagnosis (patid, encounterid, enc_type, admit_date, providerid, dx, dx_type, dx_source, dx_origin, pdx, dx_poa, raw_dx_poa) +insert into diagnosis (patid, encounterid, enc_type, admit_date,dx_date, providerid, dx, dx_type, dx_source, dx_origin, pdx, dx_poa, raw_dx_poa) /* KUMC started billing with ICD10 on Oct 1, 2015. */ with icd10_transition as ( select date '2015-10-01' as cutoff from dual @@ -252,7 +253,7 @@ with icd10_transition as ( select * from diag_fact_merge where unique_row = 1 ) -select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, enc.admit_date, enc.providerid +select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, enc.admit_date, factline.start_date, enc.providerid , factline.pcori_basecode dx , factline.dx_type dxtype, CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END dx_source, From c43e32f256dafd1f5d237d466eaa2c520f09e06c Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 21 Aug 2019 12:57:21 -0500 Subject: [PATCH 478/507] Added lab_loinc_source --- Oracle/lab_result_cm.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index bd0fc0f..df952d0 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -180,6 +180,7 @@ select distinct cast(lab.LAB_RESULT_CM_ID as varchar(19)) LAB_RESULT_CM_ID , cast(null as varchar(50)) RAW_UNIT , cast(null as varchar(50)) RAW_ORDER_DEPT , lab.RAW_FACILITY_CODE RAW_FACILITY_CODE +, 'PC' as lab_loinc_source from lab_result_w_source lab / From 3a1a6676da7e70d98b03e7a4368501aea9e0a1c5 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 21 Aug 2019 13:17:26 -0500 Subject: [PATCH 479/507] Added lab_result_source column --- Oracle/lab_result_cm.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql index df952d0..6535e0c 100644 --- a/Oracle/lab_result_cm.sql +++ b/Oracle/lab_result_cm.sql @@ -181,6 +181,7 @@ select distinct cast(lab.LAB_RESULT_CM_ID as varchar(19)) LAB_RESULT_CM_ID , cast(null as varchar(50)) RAW_ORDER_DEPT , lab.RAW_FACILITY_CODE RAW_FACILITY_CODE , 'PC' as lab_loinc_source +, 'OD' as lab_result_source from lab_result_w_source lab / From 156eefc5bd1b6c605a28d11c4d6f96b8f9a16b66 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 6 Sep 2019 09:21:03 -0500 Subject: [PATCH 480/507] code to populate obs_clin table --- Oracle/obs_clin.sql | 91 +++++++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/Oracle/obs_clin.sql b/Oracle/obs_clin.sql index d030694..4309f13 100644 --- a/Oracle/obs_clin.sql +++ b/Oracle/obs_clin.sql @@ -1,36 +1,55 @@ -/** obs_gen - create the obs_clin table. -*/ -insert into cdm_status (task, start_time) select 'obs_clin', sysdate from dual -/ -BEGIN -PMN_DROPSQL('DROP TABLE obs_clin'); -END; -/ -CREATE TABLE obs_clin( - OBSCLINID varchar(50) NOT NULL, - PATID varchar(50) NOT NULL, - ENCOUNTERID varchar(50) NULL, - OBSCLIN_PROVIDERID varchar(50) NULL, - OBSCLIN_DATE date NULL, - OBSCLIN_TIME varchar(5) NULL, - OBSCLIN_TYPE varchar(2) NULL, - OBSCLIN_CODE varchar(50) NULL, - OBSCLIN_RESULT_QUAL varchar(50) NULL, - OBSCLIN_RESULT_TEXT varchar(50) NULL, - OBSCLIN_RESULT_SNOMED varchar(50) NULL, - OBSCLIN_RESULT_NUM NUMBER(18, 0) NULL, -- (8,0) - OBSCLIN_RESULT_MODIFIER varchar(2) NULL, - OBSCLIN_RESULT_UNIT varchar(50) NULL, - RAW_OBSCLIN_NAME varchar(50) NULL, - RAW_OBSCLIN_CODE varchar(50) NULL, - RAW_OBSCLIN_TYPE varchar(50) NULL, - RAW_OBSCLIN_RESULT varchar(50) NULL, - RAW_OBSCLIN_MODIFIER varchar(50) NULL, - RAW_OBSCLIN_UNIT varchar(50) NULL -) -/ -update cdm_status -set end_time = sysdate, records = (select count(*) from obs_clin) -where task = 'obs_clin' -/ -select records + 1 from cdm_status where task = 'obs_clin' \ No newline at end of file +insert into cdm_status (task, start_time) select 'obs_clin', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE pcornet_cdm.obs_clin'); +END; +/ +BEGIN +PMN_DROPSQL('pcornet_cdm.cardiolabcomponents'); +END; + +create table pcornet_cdm.cardiolabcomponents as +select distinct cor.component_id,ceap.proc_name from clarity.order_results cor +left join clarity.order_proc cop on cop.order_proc_id=cor.order_proc_id +left join clarity.clarity_eap ceap on cop.proc_id=ceap.proc_id +where ceap.proc_name like '%ECHOCARDIOGRAM%'; + +create sequence obs_clin_seq cache 2000; + +create table pcornet_cdm.obs_clin as +select obs_clin_seq.nextval obsclinid +, lab.patid +,lab.encounterid +,' ' obsclin_providerid +,lab.lab_order_date obsclin_date +,lab.result_time obsclin_time +,lab.lab_px_type obsclin_type +,lab.lab_px obsclin_code +,lab.result_qual obsclin_result_qual +,' ' obsclin_result_text +,' ' obsclin_result_snomed +,lab.result_num obsclin_result_num +,lab.result_modifier obsclin_result_modifier +,lab.result_unit obsclin_result_unit +,card.proc_name raw_obsclin_name +,' ' raw_obsclin_code +,' ' raw_obsclin_type +,lab.raw_result raw_obsclin_result +,' ' raw_obsclin_modifier +,' ' raw_obsclin_unit +from pcornet_cdm.lab_result_cm lab +left join pcornet_cdm.cardiolabcomponents card on substr(lab.raw_facility_code,18)=card.component_id; +/ + +create index obs_clin_idx on pcornet_cdm.obs_clin (PATID, ENCOUNTERID) +/ + +BEGIN +GATHER_TABLE_STATS('OBS_CLIN'); +END; +/ + +update cdm_status +set end_time = sysdate, records = (select count(*) from obs_clin) +where task = 'obs_clin' +/ From 3edce666fb24db787541c30e8936c2d052f3b888 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 6 Sep 2019 10:19:15 -0500 Subject: [PATCH 481/507] added obs_clin create table statement --- Oracle/obs_clin.sql | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/Oracle/obs_clin.sql b/Oracle/obs_clin.sql index 4309f13..3d5c598 100644 --- a/Oracle/obs_clin.sql +++ b/Oracle/obs_clin.sql @@ -16,7 +16,32 @@ where ceap.proc_name like '%ECHOCARDIOGRAM%'; create sequence obs_clin_seq cache 2000; -create table pcornet_cdm.obs_clin as +CREATE TABLE obs_clin( + OBSCLINID varchar(50) NOT NULL, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, + OBSCLIN_PROVIDERID varchar(50) NULL, + OBSCLIN_DATE date NULL, + OBSCLIN_TIME varchar(5) NULL, + OBSCLIN_TYPE varchar(2) NULL, + OBSCLIN_CODE varchar(50) NULL, + OBSCLIN_RESULT_QUAL varchar(50) NULL, + OBSCLIN_RESULT_TEXT varchar(50) NULL, + OBSCLIN_RESULT_SNOMED varchar(50) NULL, + OBSCLIN_RESULT_NUM NUMBER(18, 0) NULL, -- (8,0) + OBSCLIN_RESULT_MODIFIER varchar(2) NULL, + OBSCLIN_RESULT_UNIT varchar(50) NULL, + RAW_OBSCLIN_NAME varchar(50) NULL, + RAW_OBSCLIN_CODE varchar(50) NULL, + RAW_OBSCLIN_TYPE varchar(50) NULL, + RAW_OBSCLIN_RESULT varchar(50) NULL, + RAW_OBSCLIN_MODIFIER varchar(50) NULL, + RAW_OBSCLIN_UNIT varchar(50) NULL +); + +insert into obs_clin(obsclinid,patid,encounterid,obsclin_providerid,obsclin_date,obsclin_time,obsclin_type,obsclin_code,obsclin_result_qual, + obsclin_result_text,obsclin_result_snomed,obsclin_result_num,obsclin_result_modifier,obsclin_result_unit,raw_obsclin_name, + raw_obsclin_code,raw_obsclin_type,raw_obsclin_result,raw_obsclin_modifier,raw_obsclin_unit) select obs_clin_seq.nextval obsclinid , lab.patid ,lab.encounterid @@ -53,3 +78,5 @@ update cdm_status set end_time = sysdate, records = (select count(*) from obs_clin) where task = 'obs_clin' / + +select records + 1 from cdm_status where task = 'obs_clin' \ No newline at end of file From 52449ea8b1a19a4ef720b721cb8258e9872cc7af Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 6 Sep 2019 10:24:56 -0500 Subject: [PATCH 482/507] Updates to the egfr lab components and the result unit values has been added --- Oracle/pcornet_loader.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Oracle/pcornet_loader.sql b/Oracle/pcornet_loader.sql index 33065cd..0bb61f1 100644 --- a/Oracle/pcornet_loader.sql +++ b/Oracle/pcornet_loader.sql @@ -37,6 +37,17 @@ begin update pcornet_cdm.lab_result_cm set result_num=result_num/1000 where lab_loinc in('2862-1','26474-7'); + + update pcornet_cdm.lab_result_cm + set lab_loinc='48642-3' + where raw_facility_code like '%KUH|COMPONENT_ID:191'; + + update pcornet_cdm.lab_result_cm + set lab_loinc='48643-1' + where raw_facility_code like '%KUH|COMPONENT_ID:200'; + + update pcornet_cdm.lab_result_cm lab + set lab.result_unit = (SELECT mc.ucum_code FROM pcornet_cdm.resultunit_manualcuration mc WHERE lab.result_unit = mc.result_unit); /* Remove rows from the PRESCRIBING table where RX_* fields are null TODO: Remove this when fixed in HERON From a3af080c5f60b758c08a19cc64a35a2ec589b905 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 6 Sep 2019 11:09:24 -0500 Subject: [PATCH 483/507] Added new harvest table columns as per CDM V5 --- Oracle/harvest.sql | 15 ++++++++++++--- curated_data/harvest_local.csv | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index 85cabc4..e9fea75 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -20,6 +20,7 @@ CREATE TABLE harvest( ENR_END_DATE_MGMT varchar(2) NULL, ADMIT_DATE_MGMT varchar(2) NULL, DISCHARGE_DATE_MGMT varchar(2) NULL, + DX_DATE_MGMT varchar92) NULL, PX_DATE_MGMT varchar(2) NULL, RX_ORDER_DATE_MGMT varchar(2) NULL, RX_START_DATE_MGMT varchar(2) NULL, @@ -38,7 +39,15 @@ CREATE TABLE harvest( MEDADMIN_STOP_DATE_MGMT varchar(2) NULL, OBSCLIN_DATE_MGMT varchar(2) NULL, OBSGEN_DATE_MGMT varchar(2) NULL, - REFRESH_DEMOGRAPHIC_DATE date NULL, + ADDRESS_PERIOD_START_MGMT varchar(2) NULL, + ADDRESS_PERIOD_END_MGMT varchar(2) NULL, + VX_RECORD_DATE_MGMT varchar(2) NULL, + VX_ADMIN_DATE_MGMT varchar(2) NULL, + VX_EXP_DATE_MGMT varchar(2) NULL, + REFRESH_HASH_TOKEN_DATE varchar(2) NULL, + REFRESH_LDS_ADDRESS_HISTORY_DATE varchar(2) NULL, + REFRESH_IMMUNIZATION_DATE varchar(2) NULL, + REFRESH_DEMOGRAPHIC_DATE date NULL, REFRESH_ENROLLMENT_DATE date NULL, REFRESH_ENCOUNTER_DATE date NULL, REFRESH_DIAGNOSIS_DATE date NULL, @@ -64,7 +73,7 @@ begin execute immediate 'truncate table harvest'; INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, - BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, + BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, DX_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, DEATH_DATE_MGMT, MEDADMIN_START_DATE_MGMT, MEDADMIN_STOP_DATE_MGMT, OBSCLIN_DATE_MGMT, OBSGEN_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, @@ -72,7 +81,7 @@ INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE, REFRESH_MED_ADMIN_DATE, REFRESH_OBS_CLIN_DATE, REFRESH_PROVIDER_DATE, REFRESH_OBS_GEN_DATE) select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 4.1, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, - hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.PX_DATE_MGMT, + hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.DX_DATE_MGMT, hl.PX_DATE_MGMT, hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, hl.DEATH_DATE_MGMT, hl.MEDADMIN_START_DATE_MGMT, hl.MEDADMIN_STOP_DATE_MGMT, hl.OBSCLIN_DATE_MGMT, hl.OBSGEN_DATE_MGMT, diff --git a/curated_data/harvest_local.csv b/curated_data/harvest_local.csv index cd46ff5..db02154 100644 --- a/curated_data/harvest_local.csv +++ b/curated_data/harvest_local.csv @@ -1,2 +1,2 @@ -"DATAMART_CLAIMS","DATAMART_EHR","BIRTH_DATE_MGMT","ENR_START_DATE_MGMT","ENR_END_DATE_MGMT","ADMIT_DATE_MGMT","DISCHARGE_DATE_MGMT","PX_DATE_MGMT","RX_ORDER_DATE_MGMT","RX_START_DATE_MGMT","RX_END_DATE_MGMT","DISPENSE_DATE_MGMT","LAB_ORDER_DATE_MGMT","SPECIMEN_DATE_MGMT","RESULT_DATE_MGMT","MEASURE_DATE_MGMT","ONSET_DATE_MGMT","REPORT_DATE_MGMT","RESOLVE_DATE_MGMT","PRO_DATE_MGMT","DEATH_DATE_MGMT","MEDADMIN_START_DATE_MGMT","MEDADMIN_STOP_DATE_MGMT","OBSCLIN_DATE_MGMT","OBSGEN_DATE_MGMT" -"01","02","03","03","03","03","03","03","03","03","04","03","03","03","03","03","03","03","03","03","03","03","03","03","03" +DATAMART_CLAIMS,DATAMART_EHR,BIRTH_DATE_MGMT,ENR_START_DATE_MGMT,ENR_END_DATE_MGMT,ADMIT_DATE_MGMT,DISCHARGE_DATE_MGMT,DX_DATE_MGMT,PX_DATE_MGMT,RX_ORDER_DATE_MGMT,RX_START_DATE_MGMT,RX_END_DATE_MGMT,DISPENSE_DATE_MGMT,LAB_ORDER_DATE_MGMT,SPECIMEN_DATE_MGMT,RESULT_DATE_MGMT,MEASURE_DATE_MGMT,ONSET_DATE_MGMT,REPORT_DATE_MGMT,RESOLVE_DATE_MGMT,PRO_DATE_MGMT,DEATH_DATE_MGMT,MEDADMIN_START_DATE_MGMT,MEDADMIN_STOP_DATE_MGMT,OBSCLIN_DATE_MGMT,OBSGEN_DATE_MGMT,ADDRESS_PERIOD_START_MGMT,ADDRESS_PERIOD_END_MGMT,VX_RECORD_DATE_MGMT,VX_ADMIN_DATE_MGMT,VX_EXP_DATE_MGMT, +1,2,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01, From becbf68489ec6b939d40290e1b81ac4eb26cbef7 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 6 Sep 2019 12:18:24 -0500 Subject: [PATCH 484/507] removed address history and immunization table columns for now --- Oracle/harvest.sql | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql index e9fea75..5133d01 100644 --- a/Oracle/harvest.sql +++ b/Oracle/harvest.sql @@ -45,8 +45,8 @@ CREATE TABLE harvest( VX_ADMIN_DATE_MGMT varchar(2) NULL, VX_EXP_DATE_MGMT varchar(2) NULL, REFRESH_HASH_TOKEN_DATE varchar(2) NULL, - REFRESH_LDS_ADDRESS_HISTORY_DATE varchar(2) NULL, - REFRESH_IMMUNIZATION_DATE varchar(2) NULL, + --REFRESH_LDS_ADDRESS_HISTORY_DATE varchar(2) NULL, + --REFRESH_IMMUNIZATION_DATE varchar(2) NULL, REFRESH_DEMOGRAPHIC_DATE date NULL, REFRESH_ENROLLMENT_DATE date NULL, REFRESH_ENCOUNTER_DATE date NULL, @@ -76,15 +76,19 @@ INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, DX_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, DEATH_DATE_MGMT, MEDADMIN_START_DATE_MGMT, MEDADMIN_STOP_DATE_MGMT, - OBSCLIN_DATE_MGMT, OBSGEN_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, + OBSCLIN_DATE_MGMT, OBSGEN_DATE_MGMT, ADDRESS_PERIOD_START_MGMT, ADDRESS_PERIOD_END_MGMT,VX_RECORD_DATE_MGMT, + VX_ADMIN_DATE_MGMT,VX_EXP_DATE_MGMT,REFRESH_HASH_TOKEN_DATE, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, - REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE, REFRESH_MED_ADMIN_DATE, REFRESH_OBS_CLIN_DATE, REFRESH_PROVIDER_DATE, REFRESH_OBS_GEN_DATE) + REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE, REFRESH_MED_ADMIN_DATE, REFRESH_OBS_CLIN_DATE, REFRESH_PROVIDER_DATE, REFRESH_OBS_GEN_DATE, + REFRESH_HASH_TOKEN_DATE +) select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 4.1, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.DX_DATE_MGMT, hl.PX_DATE_MGMT, hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, hl.DEATH_DATE_MGMT, hl.MEDADMIN_START_DATE_MGMT, hl.MEDADMIN_STOP_DATE_MGMT, hl.OBSCLIN_DATE_MGMT, hl.OBSGEN_DATE_MGMT, + hl.ADDRESS_PERIOD_START_MGMT, hl.ADDRESS_PERIOD_END_MGMT, hl.VX_RECORD_DATE_MGMT, hl.VX_ADMIN_DATE_MGMT, hl.VX_EXP_DATE_MGMT, case when (select records from cdm_status where task = 'demographic') > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, case when (select records from cdm_status where task = 'enrollment') > 0 then current_date else null end REFRESH_ENROLLMENT_DATE, case when (select records from cdm_status where task = 'encounter') > 0 then current_date else null end REFRESH_ENCOUNTER_DATE, @@ -102,7 +106,8 @@ INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART case when (select records from cdm_status where task = 'med_admin') > 0 then current_date else null end REFRESH_MED_ADMIN_DATE, case when (select records from cdm_status where task = 'obs_clin') > 0 then current_date else null end REFRESH_OBS_CLIN_DATE, case when (select records from cdm_status where task = 'provider') > 0 then current_date else null end REFRESH_PROVIDER_DATE, - case when (select records from cdm_status where task = 'obs_gen') > 0 then current_date else null end REFRESH_OBS_GEN_DATE + case when (select records from cdm_status where task = 'obs_gen') > 0 then current_date else null end REFRESH_OBS_GEN_DATE, + case when (select records from cdm_status where task = 'hash_token') > 0 then current_date else null end REFRESH_HASH_TOKEN from harvest_local hl; From 7797dbc6bd60dad5755e1dee893dedd8da1ecc79 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 6 Sep 2019 14:08:43 -0500 Subject: [PATCH 485/507] Added code to populate obs_gen table with naaccr data --- Oracle/obs_gen.sql | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/Oracle/obs_gen.sql b/Oracle/obs_gen.sql index 37b8fb4..efad7d2 100644 --- a/Oracle/obs_gen.sql +++ b/Oracle/obs_gen.sql @@ -1,11 +1,13 @@ -/** obs_gen - create the obs_gen table. -*/ +/* obs_gen - create the obs_gen table.*/ insert into cdm_status (task, start_time) select 'obs_gen', sysdate from dual / BEGIN PMN_DROPSQL('DROP TABLE obs_gen'); END; / + +create sequence obs_gen_seq cache 2000; + CREATE TABLE obs_gen( OBSGENID varchar(50) NOT NULL, PATID varchar(50) NOT NULL, @@ -28,9 +30,37 @@ CREATE TABLE obs_gen( RAW_OBSGEN_RESULT varchar(50) NULL, RAW_OBSGEN_UNIT varchar(50) NULL ) + + +drop table pcornet_cdm.obsgen_naaccr; + +create table pcornet_cdm.obsgen_naaccr as +select patient_num, encounter_num,provider_id,start_date,tval_char,nval_num,substr(concept_cd, instr(concept_cd, '|') + 1, + instr(concept_cd, ':') - instr(concept_cd, '|') - 1) as code_value,concept_cd from &&i2b2_data_schema.observation_fact + where concept_cd like '%NAACCR%'; + +insert into obs_gen(obsgenid,patid,encounterid,obsgen_providerid,obsgen_date,obsgen_code,obsgen_result_text, + obsgen_result_num,obsgen_source,raw_obsgen_code) +select +obs_gen_seq.nextval obsgenid, +obs.patient_num patid, +obs.encounter_num encounterid, +obs.provider_id obsgen_providerid, +obs.start_date obsgen_date, +lc.loinc_num obsgen_code, +obs.tval_char obsgen_result_text, +obs.nval_num obsgen_result_num, +'RG' obsgen_source, +obs.concept_cd raw_obsgen_code +from pcornet_cdm.obsgen_naaccr obs +left join pcornet_cdm.loinc_naaccr lc on obs.code_value =lc.code_value +; + +create index obs_gen_idx on obs_gen(PATID, ENCOUNTERID); / update cdm_status set end_time = sysdate, records = (select count(*) from obs_gen) where task = 'obs_gen' / -select records + 1 from cdm_status where task = 'obs_gen' \ No newline at end of file +select records + 1 from cdm_status where task = 'obs_gen'; + From ad954082c6181934f281303d4451570d530086fe Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 10 Sep 2019 11:26:38 -0500 Subject: [PATCH 486/507] Added create table statement for hash_token --- Oracle/hash_token.sql | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 Oracle/hash_token.sql diff --git a/Oracle/hash_token.sql b/Oracle/hash_token.sql new file mode 100644 index 0000000..2ef63f0 --- /dev/null +++ b/Oracle/hash_token.sql @@ -0,0 +1,31 @@ +/** harvest - create and populate the hash_token table. +*/ +insert into cdm_status (task, start_time) select 'hash_token', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE hash_token'); +END; +/ + +create table HASH_TOKEN ( +PATID varchar(50) NOT NULL, +TOKEN_01 varchar(50), +TOKEN_02 varchar(50), +TOKEN_05 varchar(50), +TOKEN_12 varchar(50), +TOKEN_17 varchar(50), +TOKEN_21 varchar(50), +TOKEN_22 varchar(50), +TOKEN_23 varchar(50), +CONSTRAINT pk_hash_token_pat PRIMARY KEY (patid), +CONSTRAINT fk_hash_dem_patid + FOREIGN KEY (patid) + REFERENCES demographic (patid) +); + +/ +update cdm_status +set end_time = sysdate, records = (select count(*) from hash_token) +where task = 'hash_token' +/ +select records from cdm_status where task = 'hash_token'; From 2a91e785095c4d6ecc1a43237b7e4e1e1acb25fb Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 10 Sep 2019 13:28:52 -0500 Subject: [PATCH 487/507] Added facility columns to the encounter table --- Oracle/encounter.sql | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index b4b991d..64cd2cb 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -118,6 +118,26 @@ left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instan left join payer_map pm on pm.payer_name = en.raw_payer_name_primary and (en.raw_payer_id_primary like 'O2|PAYER_PRIMARY:%' and pm.financial_class = sf.tval_char) or( pm.payer_name = en.raw_payer_name_primary and (en.raw_payer_id_primary like 'IDX|PAYER_PRIMARY:%')) / +create table encounter_w_fac_zip as +select en.* +sf.tval_char facility_location +from encounter_w_fin en +left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num and sf.source_column = 'FACILITY_ZIP' +/ +create table encounter_w_fac_id as +select en.* +, sf.tval_char facilityid +, sf.tval_char raw_facility_code +from encounter_w_fac_zip en +left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num and sf.source_column = 'FACILITY_ID' +/ +create table encounter_w_fac_type as +select en.* +, sf.tval_char facility_type +, sf.tval_char raw_facility_type +from encounter_w_fac_id en +left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num and sf.source_column = 'FACILITY_TYPE' +/ create table encounter as select cast(patid as varchar(50)) PATID @@ -127,9 +147,9 @@ select cast(patid as varchar(50)) PATID , en.discharge_date , to_char(en.discharge_date,'HH24:MI') DISCHARGE_TIME , en.providerid -, cast('NI' as varchar(3)) FACILITY_LOCATION +, en.FACILITY_LOCATION , en.enc_type -, cast('NI' as varchar(50)) FACILITYID +, en.FACILITYID , cast(en.discharge_disposition as varchar(2)) DISCHARGE_DISPOSITION , cast(en.discharge_status as varchar(2)) DISCHARGE_STATUS , en.drg @@ -137,21 +157,22 @@ select cast(patid as varchar(50)) PATID , cast(en.admitting_source as varchar(2)) ADMITTING_SOURCE , cast(en.payer_type_primary as varchar(5)) PAYER_TYPE_PRIMARY , cast(null as varchar(5)) PAYER_TYPE_SECONDARY -, cast('NI' as varchar(50)) FACILITY_TYPE +, en.FACILITY_TYPE , cast(null as varchar(50)) RAW_SITEID , cast(null as varchar(50)) RAW_ENC_TYPE , cast(null as varchar(50)) RAW_DISCHARGE_DISPOSITION , cast(null as varchar(50)) RAW_DISCHARGE_STATUS , cast(null as varchar(50)) RAW_DRG_TYPE , cast(null as varchar(50)) RAW_ADMITTING_SOURCE -, cast(null as varchar(50)) RAW_FACILITY_TYPE +, en.RAW_FACILITY_TYPE +, en.raw_facility_code , en.raw_payer_type_primary , en.raw_payer_name_primary , en.raw_payer_id_primary , cast(null as varchar(50)) RAW_PAYER_TYPE_SECONDARY , cast(null as varchar(50)) RAW_PAYER_NAME_SECONDARY , cast(null as varchar(50)) RAW_PAYER_ID_SECONDARY -from encounter_w_fin en +from encounter_w_fac_type en / create unique index encounter_pk on encounter (ENCOUNTERID) From 97be43622a2a64d1eb61a815242149ce2bb16ade Mon Sep 17 00:00:00 2001 From: Unknown Date: Thu, 19 Sep 2019 16:16:58 -0500 Subject: [PATCH 488/507] Made changes to the script to modify the source column to FACILITYTYPE --- Oracle/encounter.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 64cd2cb..f5280d1 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -136,7 +136,7 @@ select en.* , sf.tval_char facility_type , sf.tval_char raw_facility_type from encounter_w_fac_id en -left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num and sf.source_column = 'FACILITY_TYPE' +left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num and sf.source_column = 'FACILITYTYPE' / create table encounter as From 2ad0f1e188c376bd954b874470de1cf66b93a99f Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 20 Sep 2019 10:03:42 -0500 Subject: [PATCH 489/507] Added drop table statements for the intermediate encounter tables --- Oracle/encounter.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index f5280d1..82027a8 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -38,6 +38,20 @@ PMN_DROPSQL('drop table encounter_w_fin'); END; / +BEGIN +PMN_DROPSQL('drop table encounter_w_fac_zip'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table encounter_w_fac_id'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table encounter_w_fac_type'); +END; +/ create table drg as select * from (select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from From 7981a3b405019e212376b91ac0e4f385b5c7e1e9 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 20 Sep 2019 13:01:07 -0500 Subject: [PATCH 490/507] Removed the select statements from the cdm_status table as hash_token doesnt have any records in it --- Oracle/hash_token.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Oracle/hash_token.sql b/Oracle/hash_token.sql index 2ef63f0..11c391a 100644 --- a/Oracle/hash_token.sql +++ b/Oracle/hash_token.sql @@ -27,5 +27,4 @@ CONSTRAINT fk_hash_dem_patid update cdm_status set end_time = sysdate, records = (select count(*) from hash_token) where task = 'hash_token' -/ -select records from cdm_status where task = 'hash_token'; +/ \ No newline at end of file From f17dce7b7201fe3ec894f063839bb3e3a8c2afc2 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 20 Sep 2019 13:44:31 -0500 Subject: [PATCH 491/507] Added missing space after comma --- Oracle/diagnosis.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 8e628c7..96b9716 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -178,7 +178,7 @@ insert into poafact execute immediate 'create index poafact_idx on poafact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; GATHER_TABLE_STATS('POAFACT'); -insert into diagnosis (patid, encounterid, enc_type, admit_date,dx_date, providerid, dx, dx_type, dx_source, dx_origin, pdx, dx_poa, raw_dx_poa) +insert into diagnosis (patid, encounterid, enc_type, admit_date, dx_date, providerid, dx, dx_type, dx_source, dx_origin, pdx, dx_poa, raw_dx_poa) /* KUMC started billing with ICD10 on Oct 1, 2015. */ with icd10_transition as ( select date '2015-10-01' as cutoff from dual From 621a160253ea93850e63eb2bb0b5f162de6c73c5 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 20 Sep 2019 13:54:52 -0500 Subject: [PATCH 492/507] Added ; after select --- Oracle/diagnosis.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 96b9716..405c0a4 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -309,4 +309,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from diagnosis) where task = 'diagnosis' / -select records from cdm_status where task = 'diagnosis' \ No newline at end of file +select records from cdm_status where task = 'diagnosis'; \ No newline at end of file From d0b1e2039ea08e7aa13cee9fbe63eb18d17cd7f9 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 20 Sep 2019 13:58:00 -0500 Subject: [PATCH 493/507] Changed the indent --- Oracle/diagnosis.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index 405c0a4..cef6c09 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -12,7 +12,7 @@ CREATE TABLE diagnosis( ENCOUNTERID varchar(50) NOT NULL, ENC_TYPE varchar(2) NULL, ADMIT_DATE date NULL, - DX_DATE date NULL, + DX_DATE date NULL, PROVIDERID varchar(50) NULL, DX varchar(18) NOT NULL, DX_TYPE varchar(2) NOT NULL, From 3cb4fe7295ad1dffad543543800bf9cd1841fe4e Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 20 Sep 2019 15:18:00 -0500 Subject: [PATCH 494/507] Song consolidated a new payer_map.csv file to include IDX payers --- curated_data/payer_map.csv | 687 +++++++++++++++++-------------------- 1 file changed, 322 insertions(+), 365 deletions(-) diff --git a/curated_data/payer_map.csv b/curated_data/payer_map.csv index 42eefbe..7364238 100644 --- a/curated_data/payer_map.csv +++ b/curated_data/payer_map.csv @@ -1,365 +1,322 @@ -PAYER_NAME,FINANCIAL_CLASS,CODE,DESCRIPTIVE_TEXT -AETNA USHC HMO/POS A,AETNA,5,PRIVATE HEALTH INSURANCE -AETNA USHC HMO/POS B,AETNA,5,PRIVATE HEALTH INSURANCE -AETNA USHC PPO/TRAD A,AETNA,5,PRIVATE HEALTH INSURANCE -AETNA USHC PPO/TRAD B,AETNA,5,PRIVATE HEALTH INSURANCE -BLUE CROSS BLUE SHIELD,AGENCY,6,BLUE CROSS/BLUE SHIELD -CRIME VICTIMS KS,AGENCY,349,Other -CRIMES VICTIMS MO,AGENCY,349,Other -DEPT OF SOCIAL REHAB,AGENCY,349,Other -DISABILITY DETERMINATIONS,AGENCY,349,Other -MUSCULAR DYSTROPHY ASSOC,AGENCY,349,Other -ALT BCBS NEW DIRECTIONS,BCBS,6,BLUE CROSS/BLUE SHIELD -BCBS KANSAS,BCBS,6,BLUE CROSS/BLUE SHIELD -BCBS KC,BCBS,6,BLUE CROSS/BLUE SHIELD -BCBS KC ALTERNATE,BCBS,6,BLUE CROSS/BLUE SHIELD -EMPIRE PLAN/NYSHIP,BCBS,6,BLUE CROSS/BLUE SHIELD -INACTIVE NEW DIRECTIONS,BCBS,6,BLUE CROSS/BLUE SHIELD -PHP,BCBS,6,BLUE CROSS/BLUE SHIELD -UKH SP BCBS,BCBS,6,BLUE CROSS/BLUE SHIELD -BCBS KC HMO/POS A,BCBS KC,6,BLUE CROSS/BLUE SHIELD -BCBS KC HMO/POS B,BCBS KC,6,BLUE CROSS/BLUE SHIELD -BCBS KC INDEMNITY/OTHER,BCBS KC,6,BLUE CROSS/BLUE SHIELD -BCBS KC PPO/TRAD A,BCBS KC,6,BLUE CROSS/BLUE SHIELD -BCBS KC PPO/TRAD B,BCBS KC,6,BLUE CROSS/BLUE SHIELD -BCBS KS PREFERRED CARE OTHER,BCBS KC,6,BLUE CROSS/BLUE SHIELD -BCBS KS HMO/POS A ,BCBS KS,6,BLUE CROSS/BLUE SHIELD -BCBS KS HMO/POS B,BCBS KS,6,BLUE CROSS/BLUE SHIELD -BCBS KS PPO/TRAD A,BCBS KS,6,BLUE CROSS/BLUE SHIELD -BCBS KS PPO/TRAD B,BCBS KS,6,BLUE CROSS/BLUE SHIELD -BLUE CROSS KANSAS,BLUE CROSS KANSAS,6,BLUE CROSS/BLUE SHIELD -BLUE SHIELD,Blue Shield,6,BLUE CROSS/BLUE SHIELD -CIGNA HMO/POS A,CIGNA,5,PRIVATE HEALTH INSURANCE -CIGNA HMO/POS B,CIGNA,5,PRIVATE HEALTH INSURANCE -CIGNA PPO/TRAD A,CIGNA,5,PRIVATE HEALTH INSURANCE -CIGNA PPO/TRAD B,CIGNA,5,PRIVATE HEALTH INSURANCE -COVENTRY/PRINCIPAL HMO/POS A,COVENTRY,5,PRIVATE HEALTH INSURANCE -COVENTRY/PRINCIPAL HMO/POS B,COVENTRY,5,PRIVATE HEALTH INSURANCE -COVENTRY/PRINCIPAL PPO A,COVENTRY,5,PRIVATE HEALTH INSURANCE -COVENTRY/PRINCIPAL PPO B,COVENTRY,5,PRIVATE HEALTH INSURANCE -AETNA,Commercial,5,PRIVATE HEALTH INSURANCE -AHA-HEALTHCARE PREFERRED,Commercial,5,PRIVATE HEALTH INSURANCE -ALLEGIANCE BENEFIT PLAN MGMT INC,Commercial,5,PRIVATE HEALTH INSURANCE -ALLSTATE INSURANCE CO,Commercial,5,PRIVATE HEALTH INSURANCE -ALT COMPSYCH,Commercial,5,PRIVATE HEALTH INSURANCE -ALT COVENTRY MHNET ,Commercial,5,PRIVATE HEALTH INSURANCE -ALT GEHA,Commercial,5,PRIVATE HEALTH INSURANCE -ALT NEW DIRECTIONS,Commercial,5,PRIVATE HEALTH INSURANCE -ALT OPTUM BEHAVIORIAL HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE -ALT VALUEOPTIONS,Commercial,5,PRIVATE HEALTH INSURANCE -AMBETTER,Commercial,5,PRIVATE HEALTH INSURANCE -AMERICAN CONTINENTAL INS,Commercial,5,PRIVATE HEALTH INSURANCE -AMERICAN FAMILY LIFE INSURANCE,Commercial,5,PRIVATE HEALTH INSURANCE -AMERICAN GENERAL LIFE,Commercial,5,PRIVATE HEALTH INSURANCE -AMERICAN NATIONAL LIFE,Commercial,5,PRIVATE HEALTH INSURANCE -AMERICAN REPUBLIC INS,Commercial,5,PRIVATE HEALTH INSURANCE -AMERICAN REPUBLIC INSURANCE,Commercial,5,PRIVATE HEALTH INSURANCE -AMERICAN UNITEDLIFE,Commercial,5,PRIVATE HEALTH INSURANCE -ASSURANT HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE -BAKERY & CONFECTIONARY UNION,Commercial,5,PRIVATE HEALTH INSURANCE -BANKERS LIFE & CASUALTY,Commercial,5,PRIVATE HEALTH INSURANCE -BANKERS LIFE AND CASUALTY,Commercial,5,PRIVATE HEALTH INSURANCE -BENEFIT ADMIN SYSTEM,Commercial,5,PRIVATE HEALTH INSURANCE -BENEFIT MANAGEMENT INC,Commercial,5,PRIVATE HEALTH INSURANCE -BOILERMAKERS,Commercial,5,PRIVATE HEALTH INSURANCE -CARE SUPPLEMENT,Commercial,5,PRIVATE HEALTH INSURANCE -CENTRAL RESERVE LIFE,Commercial,5,PRIVATE HEALTH INSURANCE -CENTURY HEALTH SOLUTIONS,Commercial,5,PRIVATE HEALTH INSURANCE -CERNER,Commercial,5,PRIVATE HEALTH INSURANCE -CHAMPVA,Commercial,3221,Civilian Health and Medical Program for the VA (CHAMPVA) -CHESTERFIELD RESOURCES,Commercial,5,PRIVATE HEALTH INSURANCE -CHRISTIAN FIDELITY,Commercial,5,PRIVATE HEALTH INSURANCE -CHUBB GROUP OF INSURANCE,Commercial,5,PRIVATE HEALTH INSURANCE -CIGNA,Commercial,5,PRIVATE HEALTH INSURANCE -CONTINENTAL GENERAL INS,Commercial,5,PRIVATE HEALTH INSURANCE -CONTINENTAL LIFE OF BRENTWOOD,Commercial,5,PRIVATE HEALTH INSURANCE -CORESOURCE,Commercial,5,PRIVATE HEALTH INSURANCE -CORIZON,Commercial,5,PRIVATE HEALTH INSURANCE -CORRECT CARE SOLUTIONS,Commercial,5,PRIVATE HEALTH INSURANCE -COVENTRY,Commercial,5,PRIVATE HEALTH INSURANCE -CRIME VICTIM KS,Commercial,5,PRIVATE HEALTH INSURANCE -CRIME VICTIM MO,Commercial,5,PRIVATE HEALTH INSURANCE -DISABILITY DETERM,Commercial,5,PRIVATE HEALTH INSURANCE -EMPLOYERS HEALTH NETWORK,Commercial,5,PRIVATE HEALTH INSURANCE -EQUITABLE LIFE & CASUALTY,Commercial,5,PRIVATE HEALTH INSURANCE -FARMERS INSURANCE,Commercial,5,PRIVATE HEALTH INSURANCE -FIDELITY,Commercial,5,PRIVATE HEALTH INSURANCE -FIRST HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE -FORTIS,Commercial,5,PRIVATE HEALTH INSURANCE -FRA MILICARE PLUS,Commercial,5,PRIVATE HEALTH INSURANCE -GALLAGHER BENEFIT ADMIN,Commercial,5,PRIVATE HEALTH INSURANCE -GEHA,Commercial,5,PRIVATE HEALTH INSURANCE -GEICO,Commercial,5,PRIVATE HEALTH INSURANCE -GENERIC COMMERCIAL,Commercial,5,PRIVATE HEALTH INSURANCE -GENERIC TRANSPLANT,Commercial,5,PRIVATE HEALTH INSURANCE -GPA,Commercial,5,PRIVATE HEALTH INSURANCE -HCH ADMINISTRATION,Commercial,5,PRIVATE HEALTH INSURANCE -HEALTH COST SOLUTIONS,Commercial,5,PRIVATE HEALTH INSURANCE -HEALTHLINK,Commercial,5,PRIVATE HEALTH INSURANCE -HUMANA,Commercial,5,PRIVATE HEALTH INSURANCE -IBEW LOCAL UNION 124 H&W,Commercial,5,PRIVATE HEALTH INSURANCE -INACTIVE-TRIWEST,Commercial,5,PRIVATE HEALTH INSURANCE -INDIAN HEALTH SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE -KAISER PERMANENTE,Commercial,5,PRIVATE HEALTH INSURANCE -KANSAS CITY HOSPICE,Commercial,5,PRIVATE HEALTH INSURANCE -KENDALLWOOD HOSPICE,Commercial,5,PRIVATE HEALTH INSURANCE -KU ATHLETICS,Commercial,5,PRIVATE HEALTH INSURANCE -KU RX INTERNAL,Commercial,5,PRIVATE HEALTH INSURANCE -KU RX WAM RHTEST,Commercial,5,PRIVATE HEALTH INSURANCE -LARNED STATE HOSPITAL,Commercial,5,PRIVATE HEALTH INSURANCE -LEAGUE MEDICAL,Commercial,5,PRIVATE HEALTH INSURANCE -LIFETRAC TRANSPLANT,Commercial,5,PRIVATE HEALTH INSURANCE -LINKIA,Commercial,5,PRIVATE HEALTH INSURANCE -MEDICA,Commercial,5,PRIVATE HEALTH INSURANCE -MEDICAL EXCESS TRANSPLANT,Commercial,5,PRIVATE HEALTH INSURANCE -MEDICO INSURANCE CO,Commercial,5,PRIVATE HEALTH INSURANCE -MEGA LIFE & HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE -MERITAIN HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE -MH NET,Commercial,5,PRIVATE HEALTH INSURANCE -MOAA MEDIPLUS,Commercial,5,PRIVATE HEALTH INSURANCE -MODEL MC PAYOR,Commercial,5,PRIVATE HEALTH INSURANCE -MODEL RTE PAYOR,Commercial,5,PRIVATE HEALTH INSURANCE -MULTIPLAN,Commercial,5,PRIVATE HEALTH INSURANCE -MUTUAL OF OMAHA,Commercial,5,PRIVATE HEALTH INSURANCE -NATIONAL ASSN LETTER CARRIERS,Commercial,5,PRIVATE HEALTH INSURANCE -NECA IBEW FAMILY MEDICAL PLAN,Commercial,5,PRIVATE HEALTH INSURANCE -NORTH AMERICAN INS,Commercial,5,PRIVATE HEALTH INSURANCE -OLD SURETY LIFE INSURANCE CO,Commercial,5,PRIVATE HEALTH INSURANCE -OPTUMHEALTH,Commercial,5,PRIVATE HEALTH INSURANCE -ORSCHELN INDUSTRIES,Commercial,5,PRIVATE HEALTH INSURANCE -PHYSICIANS MUTUAL,Commercial,5,PRIVATE HEALTH INSURANCE -PLAN CARE AMERICA,Commercial,5,PRIVATE HEALTH INSURANCE -PLUMBERS LOCAL 8,Commercial,5,PRIVATE HEALTH INSURANCE -PRINCIPAL FINANCIAL GROUP,Commercial,5,PRIVATE HEALTH INSURANCE -PRISON HEALTH SYSTEMS,Commercial,5,PRIVATE HEALTH INSURANCE -PYRAMID LIFE,Commercial,5,PRIVATE HEALTH INSURANCE -QUIK TRIP,Commercial,5,PRIVATE HEALTH INSURANCE -REI,Commercial,5,PRIVATE HEALTH INSURANCE -RESERVE NATIONAL,Commercial,5,PRIVATE HEALTH INSURANCE -RX ADVANCE PCS,Commercial,5,PRIVATE HEALTH INSURANCE -RX AETNA,Commercial,5,PRIVATE HEALTH INSURANCE -RX ALLWIN DATA,Commercial,5,PRIVATE HEALTH INSURANCE -RX ALPHASCRIP,Commercial,5,PRIVATE HEALTH INSURANCE -RX ARGUS HEALTH SYSTEMS,Commercial,5,PRIVATE HEALTH INSURANCE -RX CATALYST,Commercial,5,PRIVATE HEALTH INSURANCE -RX CIGNA,Commercial,5,PRIVATE HEALTH INSURANCE -RX CVS/CAREMARK,Commercial,5,PRIVATE HEALTH INSURANCE -RX EMDEON,Commercial,5,PRIVATE HEALTH INSURANCE -RX EMPLOYEE BENEFITS MANAGEMENT SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE -RX ENVISIONRX,Commercial,5,PRIVATE HEALTH INSURANCE -RX EXPRESS SCRIPTS,Commercial,5,PRIVATE HEALTH INSURANCE -RX HEALTHTRANS,Commercial,5,PRIVATE HEALTH INSURANCE -RX HUMANA,Commercial,5,PRIVATE HEALTH INSURANCE -RX LDI INTEGRATED PHARMACY SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE -RX MAGELLAN,Commercial,5,PRIVATE HEALTH INSURANCE -RX MAXOR PLUS,Commercial,5,PRIVATE HEALTH INSURANCE -RX MEDCO,Commercial,5,PRIVATE HEALTH INSURANCE -RX MEDIMPACT,Commercial,5,PRIVATE HEALTH INSURANCE -RX MEDTRAK,Commercial,5,PRIVATE HEALTH INSURANCE -RX MO MEDICAID,Commercial,5,PRIVATE HEALTH INSURANCE -RX NATIONAL PRESCRIPTION SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE -RX NETCARD SYSTEMS,Commercial,5,PRIVATE HEALTH INSURANCE -RX OPTUM RX,Commercial,5,PRIVATE HEALTH INSURANCE -RX PBM PLUS,Commercial,5,PRIVATE HEALTH INSURANCE -RX PHARMACY DATA MANAGEMENT,Commercial,5,PRIVATE HEALTH INSURANCE -RX PMSI,Commercial,5,PRIVATE HEALTH INSURANCE -RX PRIME THERAPEUTICS,Commercial,5,PRIVATE HEALTH INSURANCE -RX PROCARE,Commercial,5,PRIVATE HEALTH INSURANCE -RX RESTAT,Commercial,5,PRIVATE HEALTH INSURANCE -RX RYAN WHITE,Commercial,5,PRIVATE HEALTH INSURANCE -RX SAV-RX,Commercial,5,PRIVATE HEALTH INSURANCE -RX THERAPY FIRST,Commercial,5,PRIVATE HEALTH INSURANCE -RX TMESYS,Commercial,5,PRIVATE HEALTH INSURANCE -RX UNITEDHEALTHCARE,Commercial,5,PRIVATE HEALTH INSURANCE -RX US SCRIPT,Commercial,5,PRIVATE HEALTH INSURANCE -SANFORD HEALTH PLAN,Commercial,5,PRIVATE HEALTH INSURANCE -SEECHANGE HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE -SELMAN AND COMPANY,Commercial,5,PRIVATE HEALTH INSURANCE -SHEET METAL WORKERS NATIONAL,Commercial,5,PRIVATE HEALTH INSURANCE -STANDARD LIFE & ACCIDENT,Commercial,5,PRIVATE HEALTH INSURANCE -STAR HRG/STAR ADMIN SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE -STARMARK,Commercial,5,PRIVATE HEALTH INSURANCE -STATE FARM INS,Commercial,5,PRIVATE HEALTH INSURANCE -STATE FARM INSURANCE,Commercial,5,PRIVATE HEALTH INSURANCE -STATE OF KANSAS,Commercial,5,PRIVATE HEALTH INSURANCE -STATE OF MISSOURI,Commercial,5,PRIVATE HEALTH INSURANCE -STERLING LIFE INS,Commercial,5,PRIVATE HEALTH INSURANCE -STUDENT ASSURANCE SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE -THE EPOCH GROUP,Commercial,5,PRIVATE HEALTH INSURANCE -THREE RIVERS PROVIDER NETWORK,Commercial,5,PRIVATE HEALTH INSURANCE -THRIVENT FOR LUTHERANS,Commercial,5,PRIVATE HEALTH INSURANCE -TRANSAMERICA LIFE INS CO,Commercial,5,PRIVATE HEALTH INSURANCE -UHC,Commercial,5,PRIVATE HEALTH INSURANCE -UHC EMPIRE/NYSHIP PB ALT,Commercial,5,PRIVATE HEALTH INSURANCE -UKH SP AETNA,Commercial,5,PRIVATE HEALTH INSURANCE -UKH SP CIGNA,Commercial,5,PRIVATE HEALTH INSURANCE -UKH SP COVENTRY,Commercial,5,PRIVATE HEALTH INSURANCE -UKH SP EOB TO 835,Commercial,5,PRIVATE HEALTH INSURANCE -UKH SP GEHA,Commercial,5,PRIVATE HEALTH INSURANCE -UKH SP GENERIC ,Commercial,5,PRIVATE HEALTH INSURANCE -UKH SP HUMANA,Commercial,5,PRIVATE HEALTH INSURANCE -UKH SP OPTUM,Commercial,5,PRIVATE HEALTH INSURANCE -UKH SP UHC,Commercial,5,PRIVATE HEALTH INSURANCE -UNITED AMERICAN INS,Commercial,5,PRIVATE HEALTH INSURANCE -UPREHS,Commercial,5,PRIVATE HEALTH INSURANCE -US MARSHALL SERVICE,Commercial,5,PRIVATE HEALTH INSURANCE -USA MANAGED CARE,Commercial,5,PRIVATE HEALTH INSURANCE -USAA LIFE INS CO,Commercial,5,PRIVATE HEALTH INSURANCE -VISION,Commercial,5,PRIVATE HEALTH INSURANCE -WILSON-MCSHANE CORP,Commercial,5,PRIVATE HEALTH INSURANCE -WPPA,Commercial,5,PRIVATE HEALTH INSURANCE -GRANT/STUDY,GRANT/STUDY,5,PRIVATE HEALTH INSURANCE -HUMANA HMO A,HUMANA,5,PRIVATE HEALTH INSURANCE -HUMANA HMO B,HUMANA,5,PRIVATE HEALTH INSURANCE -HUMANA PPO A,HUMANA,5,PRIVATE HEALTH INSURANCE -HUMANA PPO B,HUMANA,5,PRIVATE HEALTH INSURANCE -FAMILY HEALTH PARTNERS,MEDICAID KS,2,MEDICAID -KS MEDICAID,MEDICAID KS,2,MEDICAID -KS MEDICAID HMO B,MEDICAID KS,2,MEDICAID -MEDICAID KS,MEDICAID KS,2,MEDICAID -MEDICAID KS HOM(102 REPLACEMEN,MEDICAID KS,2,MEDICAID -UKH SP KS MEDICAID,MEDICAID KS,2,MEDICAID -MEDICAID MO,MEDICAID MO,2,MEDICAID -MEDICAID MO HMO,MEDICAID MO,2,MEDICAID -MEDICAID OUT OF STATE,Medicaid OOS,25,Medicaid - Out of State -KS MEDICAID PENDING,MEDICAID PENDING,2,MEDICAID -MO MEDICAID PENDING,MEDICAID PENDING,2,MEDICAID -MEDICAID,Medicaid,2,MEDICAID -ALT MO MEDICAID,Medicaid OOS,25,Medicaid - Out of State -MO MEDICAID,Medicaid OOS,25,Medicaid - Out of State -UKH SP OOS MEDICAID,Medicaid OOS,25,Medicaid - Out of State -ALT CENPATICO KS MCAID BEHAVIORAL,Medicaid Repl KS,2,MEDICAID -AMERIGROUP MEDICAID KS,Medicaid Repl KS,2,MEDICAID -CENTENE MEDICAID KS,Medicaid Repl KS,2,MEDICAID -MEDICAID OTHER HMO KS GENERIC,Medicaid Repl KS,2,MEDICAID -UHC MEDICAID KS,Medicaid Repl KS,2,MEDICAID -VIA CHRISTI HOPE,Medicaid Repl KS,2,MEDICAID -AETNA MEDICAID OOS,Medicaid Repl OOS,25,Medicaid - Out of State -ALT CENPATICO MO MCAID BEHAVIORAL,Medicaid Repl OOS,25,Medicaid - Out of State -COVENTRY MEDICAID OOS,Medicaid Repl OOS,25,Medicaid - Out of State -HOME STATE HEALTH PLAN,Medicaid Repl OOS,25,Medicaid - Out of State -MEDICAID OTHER HMO OOS GENERIC,Medicaid Repl OOS,25,Medicaid - Out of State -MISSOURI CARE,Medicaid Repl OOS,25,Medicaid - Out of State -UHC MEDICAID MO,Medicaid Repl OOS,25,Medicaid - Out of State -ALT MEDICARE,Medicare,1,MEDICARE -ALT MEDICARE-NORIDIAN,Medicare,1,MEDICARE -CARE,Medicare,1,MEDICARE -DMERC,Medicare,1,MEDICARE -MEDICARE,Medicare,1,MEDICARE -MEDICARE HMO,Medicare,1,MEDICARE -MEDICARE HMO B,Medicare,1,MEDICARE -MEDICARE PPO A,Medicare,1,MEDICARE -MEDICARE RAILROAD,Medicare,1,MEDICARE -UKH SP MEDICARE,Medicare,1,MEDICARE -AETNA MEDICARE,Medicare Repl,1,MEDICARE -ALLWELL (OON),Medicare Repl,1,MEDICARE -ALT COVENTRY ADVANTRA MHNET,Medicare Repl,1,MEDICARE -BCBS KC MEDICARE,Medicare Repl,1,MEDICARE -CARE IMPROVEMENT PLUS,Medicare Repl,1,MEDICARE -CHILDREN'S MERCY HOSPITAL,Medicare Repl,1,MEDICARE -CIGNA MEDICARE,Medicare Repl,1,MEDICARE -CONSOLIDATED BILLING,Medicare Repl,1,MEDICARE -COVENTRY MEDICARE,Medicare Repl,1,MEDICARE -HUMANA MEDICARE,Medicare Repl,1,MEDICARE -MEDICARE REPL GENERIC,Medicare Repl,1,MEDICARE -TODAY'S OPTIONS,Medicare Repl,1,MEDICARE -UHC MEDICARE,Medicare Repl,1,MEDICARE -AUTO/LIABILITY,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE -GLOBAL TRANSPLANTS,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE -KUH EMPL SUPPLEMENTAL PLAN,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE -LIABILITY OTHER (NOT AUTO/WC),OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE -LITIGATION/LEGAL ,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE -OTHER/COMMERCIAL HMO/POS A,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE -OTHER/COMMERCIAL HMO/POS B,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE -OTHER/COMMERCIAL PPO/TRAD A,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE -OTHER/COMMERCIAL PPO/TRAD B,OTHER COMMERCIAL INSURANCE,5,PRIVATE HEALTH INSURANCE -AGENCY,Other,349,Other -METRO CARE,Other,349,Other -NORTHLAND CARE,Other,349,Other -PROJECT EAGLE,Other,349,Other -STUDY,Other,349,Other -WYJO CARE,Other,349,Other -ELECTIVE PACKAGE RATE,Self-pay,81,Self-pay -PATIENT REQUEST-DON'T BILL INS,Self-pay,81,Self-pay -SELF PAY,Self-pay,81,Self-pay -HEALTH NET FEDERAL SERVICES,Tricare,3113,TRICARE Standard - Fee For Service -TRICARE,Tricare,3113,TRICARE Standard - Fee For Service -TRICARE HMO,Tricare,3111,TRICARE Prime?HMO -TRICARE PPO/TRAD,Tricare,3112,TRICARE Extra?PPO -UKH SP TRICARE,Tricare,3113,TRICARE Standard - Fee For Service -UNITED HEALTH/METRA HMO/POS A,UNITED HEALTH,5,PRIVATE HEALTH INSURANCE -UNITED HEALTH/METRA HMO/POS B,UNITED HEALTH,5,PRIVATE HEALTH INSURANCE -UNITED HEALTH/METRA PPO/TRAD A,UNITED HEALTH,5,PRIVATE HEALTH INSURANCE -UNITED HEALTH/METRA PPO/TRAD B,UNITED HEALTH,5,PRIVATE HEALTH INSURANCE -CHAMPVA,VA,3221,Civilian Health and Medical Program for the VA (CHAMPVA) -TRIWEST,VA,32,Department of Veterans Affairs -UKH SP CHAMPVA/VA,VA,32,Department of Veterans Affairs -VA,VA,32,Department of Veterans Affairs -VETERANS ADMINISTRATION,VA,32,Department of Veterans Affairs -GALLAGHER BASSETT,Worker's Comp,95,Worker's Compensation -GENERIC WORKERS COMP,Worker's Comp,95,Worker's Compensation -MS WORKERS COMP,Worker's Comp,95,Worker's Compensation -WORKERS COMP,Worker's Comp,95,Worker's Compensation -WORKMAN'S COMPENSATION,Worker's Comp,95,Worker's Compensation -AIH COLLECTIONS,,349,Other -AMERIGROUP KS A,,38,"Other Government (Federal, State, Local not specified)" -AMERIGROUP KS B,,38,"Other Government (Federal, State, Local not specified)" -BAD DEBT,,349,Other -BANKRUPTCY,,349,Other -BCBS KC APPEALS-KUPI,,6,BLUE CROSS/BLUE SHIELD -BCBS KC APPEALS-PER SE,,6,BLUE CROSS/BLUE SHIELD -BCBS KC PREFERRED CARE OTHER,,6,BLUE CROSS/BLUE SHIELD -BCBS KS APPEALS-KUPI,,6,BLUE CROSS/BLUE SHIELD -BCBS KS APPEALS-PER SE,,6,BLUE CROSS/BLUE SHIELD -BCBS KS HMO/POS A,,6,BLUE CROSS/BLUE SHIELD -BERLIN WHEELER COLLECTIONS,,349,Other -BERMAN & RABIN COLLECTIONS,,349,Other -BUDGET PLAN,,349,Other -CHARITY/HARDSHIP,,821,Charity -COLLECTION AGENCY,,349,Other -CONSUMER COLL MGMT COLLECTIONS,,349,Other -CORPORATE ACCOUNTS,,349,Other -CREDIT LETTER SENT,,349,Other -CRIME VICTIMS MO,,349,Other -CSR COLLECTIONS,,349,Other -DENTAL A,,561,Dental -DO NOT BILL,,349,Other -DO NOT DUNN,,349,Other -ELECTIVE PACKAGE RATE A,,349,Other -FHP APPEAL-PER SE,,349,Other -FHP APPEALS-KUPI,,349,Other -FOREIGN MAIL,,349,Other -KANCARE BEHAVIORAL HLTH A,,38,"Other Government (Federal, State, Local not specified)" -KANCARE BEHAVIORAL HLTH B,,38,"Other Government (Federal, State, Local not specified)" -KUPI ADMINISTRATIVE,,349,Other -KUPI COMPLIANCE,,349,Other -LITIGATION/LEGAL,,349,Other -LIVING DONOR BALANCE,,349,Other -MEDICAID KS APPEALS-KUPI,,2,MEDICAID -MEDICAID KS APPEALS-PER SE,,2,MEDICAID -MEDICAID KS HMO,,2,MEDICAID -MEDICAID KS HMO (102 REPLACEMENT),,2,MEDICAID -MEDICAID MO APPEALS-KUPI,,2,MEDICAID -MEDICAID MO APPEALS-PER SE,,2,MEDICAID -MEDICAID MO HMO B,,2,MEDICAID -MEDICARE APPEALS-KUPI,,1,MEDICARE -MEDICARE APPEALS-PER SE,,1,MEDICARE -MEDICARE RR APPEALS-KUPI,,1,MEDICARE -MEDICARE RR APPEALS-PER SE,,1,MEDICARE -OTHER/COMMERCIAL APPEALS-KUPI,,349,Other -OTHER/COMMERCIAL APPEALS-PER SE,,349,Other -PATIENT REQUEST-DO NOT BILL INSURANCE,,349,Other -PER SE TURN LIST REVIEW,,349,Other -RECONCILIATION,,349,Other -RETURN MAIL/BAD ADDRESS,,349,Other -RTN MAIL AFTER TURN LIST RVW,,349,Other -SUNFLOWER STATE KS A,,2,MEDICAID -SUNFLOWER STATE KS B,,2,MEDICAID -SUSPENDED CLAIMS,,349,Other -TRICARE UHC HMO A,,3111,TRICARE Prime?HMO -TRICARE UHC HMO B,,3111,TRICARE Prime?HMO -TRICARE UHC PPO A,,3112,TRICARE Extra?PPO -TRICARE UHC PPO B,,3112,TRICARE Extra?PPO -TURN LIST/CREDIT REVIEW,,349,Other -UHC COMMUNITY PLAN KS A,,5,PRIVATE HEALTH INSURANCE -UHC COMMUNITY PLAN KS B,,5,PRIVATE HEALTH INSURANCE +PAYER_NAME,FINANCIAL_CLASS,CODE,DESCRIPTIVE_TEXT,SOURCE SYSTEM +AIH COLLECTIONS,@,8,NO PAYMENT from an Organization/Agency/Program/Private Payer Listed,IDX +BANKRUPTCY,@,83,Refusal to Pay/Bad Debt,IDX +BERLIN WHEELER COLLECTIONS,@,8,NO PAYMENT from an Organization/Agency/Program/Private Payer Listed,IDX +BERMAN & RABIN COLLECTIONS,@,8,NO PAYMENT from an Organization/Agency/Program/Private Payer Listed,IDX +BUDGET PLAN,@,OT,Other,IDX +COLLECTION AGENCY,@,8,NO PAYMENT from an Organization/Agency/Program/Private Payer Listed,IDX +CONSUMER COLL MGMT COLLECTIONS,@,OT,Other,IDX +CREDIT LETTER SENT,@,OT,Other,IDX +CRIME VICTIMS KS,@,37,Local Government,IDX +CRIME VICTIMS MO,@,37,Local Government,IDX +CSR COLLECTIONS,@,2,MEDICAID,IDX +DEPT OF SOCIAL REHAB,@,2,MEDICAID,IDX +DMERC,@,1,MEDICARE,IDX +DO NOT BILL,@,82,No Charge,IDX +DO NOT DUNN,@,82,No Charge,IDX +ELECTIVE PACKAGE RATE,@,9999,Unavailable / No Payer Specified / Blank,IDX +ELECTIVE PACKAGE RATE A,@,9999,Unavailable / No Payer Specified / Blank,IDX +GRANT/STUDY,@,823,Research/Clinical Trial,IDX +LITIGATION/LEGAL,@,97,Legal Liability / Liability Insurance,IDX +LIVING DONOR BALANCE,@,85,Research/Donor,IDX +MUSCULAR DYSTROPHY ASSOC,@,8,NO PAYMENT from an Organization/Agency/Program/Private Payer Listed,IDX +Other NOS,@,9999,MISCELLANEOUS/OTHER,UHC +PATIENT REQUEST-DO NOT BILL INSURANCE,@,OT,Other,IDX +PER SE TURN LIST REVIEW,@,OT,Other,IDX +RECONCILIATION,@,OT,Other,IDX +RETURN MAIL/BAD ADDRESS,@,OT,Other,IDX +RTN MAIL AFTER TURN LIST RVW,@,OT,Other,IDX +SUSPENDED CLAIMS,@,OT,Other,IDX +TURN LIST/CREDIT REVIEW,@,OT,Other,IDX +Unknown NOS,@,9999,NO PAYMENT from an Organization/Agency/Program/Private Payer Listed,UHC +AMERIGROUP KS A,@,2,MEDICAID,IDX +AMERIGROUP KS B,@,2,MEDICAID,IDX +FAMILY HEALTH PARTNERS,@,1,MEDICARE,IDX +GLOBAL TRANSPLANTS,@,OT,Other,IDX +KANCARE BEHAVIORAL HLTH A,@,2,MEDICAID,IDX +KANCARE BEHAVIORAL HLTH B,@,2,MEDICAID,IDX +SUNFLOWER STATE KS A,@,2,MEDICAID,IDX +SUNFLOWER STATE KS B,@,2,MEDICAID,IDX +AETNA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +AETNA MEDICAID OOS,Medicaid Repl OOS,25,Medicaid - Out of State,EPIC +AETNA MEDICARE,Medicare Repl,1,MEDICARE,EPIC +AETNA USHC HMO/POS A,@,5,PRIVATE HEALTH INSURANCE,IDX +AETNA USHC HMO/POS B,@,5,PRIVATE HEALTH INSURANCE,IDX +AETNA USHC PPO/TRAD A,@,5,PRIVATE HEALTH INSURANCE,IDX +AETNA USHC PPO/TRAD B,@,5,PRIVATE HEALTH INSURANCE,IDX +AGENCY,Other,349,Other,EPIC +AHA-HEALTHCARE PREFERRED,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +ALLEGIANCE BENEFIT PLAN MGMT INC,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +ALLWELL (OON),Medicare Repl,1,MEDICARE,EPIC +AMBETTER,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +AMERICAN CONTINENTAL INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +AMERICAN NATIONAL LIFE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +AMERICAN REPUBLIC INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +AMERIGROUP MEDICAID KS,Medicaid Repl KS,2,MEDICAID,EPIC +ASSURANT HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +BAKERY & CONFECTIONARY UNION,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +BANKERS LIFE & CASUALTY,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +BCBS KANSAS,BCBS,6,BLUE CROSS/BLUE SHIELD,EPIC +BCBS KC,BCBS,6,BLUE CROSS/BLUE SHIELD,EPIC +BCBS KC MEDICARE,Medicare Repl,1,MEDICARE,EPIC +BCBS KC APPEALS-KUPI,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC APPEALS-PER SE,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC HMO/POS A,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC HMO/POS B,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC INDEMNITY/OTHER,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC PPO/TRAD A,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC PPO/TRAD B,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC PREFERRED CARE OTHER,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KS APPEALS-KUPI,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KS APPEALS-PER SE,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KS HMO/POS A,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KS HMO/POS B,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KS PPO/TRAD A,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KS PPO/TRAD B,@,6,BLUE CROSS/BLUE SHIELD,IDX +BENEFIT ADMIN SYSTEM,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +BENEFIT MANAGEMENT INC,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +BOILERMAKERS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CARE,Medicare,1,MEDICARE,EPIC +CARE IMPROVEMENT PLUS,Medicare Repl,1,MEDICARE,EPIC +CARE SUPPLEMENT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CENTENE MEDICAID KS,Medicaid Repl KS,2,MEDICAID,EPIC +CENTRAL RESERVE LIFE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CENTURY HEALTH SOLUTIONS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CERNER,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CHAMPVA,VA,3221,Civilian Health and Medical Program for the VA (CHAMPVA),EPIC +CHAMPVA,Commercial,3221,Civilian Health and Medical Program for the VA (CHAMPVA),EPIC +CHESTERFIELD RESOURCES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CHRISTIAN FIDELITY,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CIGNA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CIGNA MEDICARE,Medicare Repl,1,MEDICARE,EPIC +CIGNA HMO/POS A,@,5,PRIVATE HEALTH INSURANCE,IDX +CIGNA HMO/POS B,@,5,PRIVATE HEALTH INSURANCE,IDX +CIGNA PPO/TRAD A,@,5,PRIVATE HEALTH INSURANCE,IDX +CIGNA PPO/TRAD B,@,5,PRIVATE HEALTH INSURANCE,IDX +CONSOLIDATED BILLING,Medicare Repl,1,MEDICARE,EPIC +CONTINENTAL GENERAL INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CONTINENTAL LIFE OF BRENTWOOD,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CORESOURCE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CORIZON,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CORRECT CARE SOLUTIONS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +COVENTRY,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +COVENTRY MEDICAID OOS,Medicaid Repl OOS,25,Medicaid - Out of State,EPIC +COVENTRY MEDICARE,Medicare Repl,1,MEDICARE,EPIC +COVENTRY/PRINCIPAL HMO/POS A,@,5,PRIVATE HEALTH INSURANCE,IDX +COVENTRY/PRINCIPAL HMO/POS B,@,5,PRIVATE HEALTH INSURANCE,IDX +COVENTRY/PRINCIPAL PPO A,@,5,PRIVATE HEALTH INSURANCE,IDX +COVENTRY/PRINCIPAL PPO B,@,5,PRIVATE HEALTH INSURANCE,IDX +CRIME VICTIM KS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CRIME VICTIM MO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +DISABILITY DETERM,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +EMPIRE PLAN/NYSHIP,BCBS,6,BLUE CROSS/BLUE SHIELD,EPIC +EMPLOYERS HEALTH NETWORK,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +EQUITABLE LIFE & CASUALTY,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +FARMERS INSURANCE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +FIRST HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +FRA MILICARE PLUS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +GALLAGHER BASSETT,Worker's Comp,95,Worker's Compensation,EPIC +GEHA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +GENERIC COMMERCIAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +GENERIC TRANSPLANT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +GPA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +HCH ADMINISTRATION,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +HEALTH COST SOLUTIONS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +HEALTH NET FEDERAL SERVICES,Tricare,3113,TRICARE Standard - Fee For Service,EPIC +HEALTHLINK,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +HOME STATE HEALTH PLAN,Medicaid Repl OOS,25,Medicaid - Out of State,EPIC +HUMANA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +HUMANA MEDICARE,Medicare Repl,1,MEDICARE,EPIC +HUMANA MILITARY,Tricare,31,Deparment of Defense ,EPIC +HUMANA HMO A,@,5,PRIVATE HEALTH INSURANCE,IDX +HUMANA HMO B,@,5,PRIVATE HEALTH INSURANCE,IDX +HUMANA PPO A,@,5,PRIVATE HEALTH INSURANCE,IDX +HUMANA PPO B,@,5,PRIVATE HEALTH INSURANCE,IDX +IBEW LOCAL UNION 124 H&W,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +INACTIVE NEW DIRECTIONS,BCBS,6,BLUE CROSS/BLUE SHIELD,EPIC +INACTIVE-TRIWEST,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +INDIAN HEALTH SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +KANSAS CITY HOSPICE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +KENDALLWOOD HOSPICE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +KS MEDICAID,MEDICAID KS,2,MEDICAID,EPIC +KS MEDICAID PENDING,MEDICAID PENDING,NI,No Information,EPIC +KS MEDICAID HMO B,@,2,MEDICAID,IDX +KU ATHLETICS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +KU RX INTERNAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +LARNED STATE HOSPITAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +LEAGUE MEDICAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +LIFETRAC TRANSPLANT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MEDICA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MEDICAID OTHER HMO KS GENERIC,Medicaid Repl KS,2,MEDICAID,EPIC +MEDICAID OTHER HMO OOS GENERIC,Medicaid Repl OOS,25,Medicaid - Out of State,EPIC +MEDICAID OUT OF STATE,Medicaid OOS,25,Medicaid - Out of State,EPIC +MEDICAL EXCESS TRANSPLANT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MEDICAID KS,@,2,MEDICAID,IDX +MEDICAID KS APPEALS-KUPI,@,2,MEDICAID,IDX +MEDICAID KS APPEALS-PER SE,@,2,MEDICAID,IDX +MEDICAID KS HMO (102 REPLACEMENT),@,2,MEDICAID,IDX +MEDICAID MO,@,2,MEDICAID,IDX +MEDICAID MO APPEALS-KUPI,@,2,MEDICAID,IDX +MEDICAID MO APPEALS-PER SE,@,2,MEDICAID,IDX +MEDICAID MO HMO,@,2,MEDICAID,IDX +MEDICAID MO HMO B,@,2,MEDICAID,IDX +MEDICARE,Medicare,1,MEDICARE,EPIC +MEDICARE RAILROAD,Medicare,1,MEDICARE,EPIC +MEDICARE REPL GENERIC,Medicare Repl,1,MEDICARE,EPIC +MEDICARE APPEALS-KUPI,@,1,MEDICARE,IDX +MEDICARE APPEALS-PER SE,@,1,MEDICARE,IDX +MEDICARE HMO,@,1,MEDICARE,IDX +MEDICARE HMO B,@,1,MEDICARE,IDX +MEDICARE PPO A,@,1,MEDICARE,IDX +MEDICARE RR APPEALS-PER SE,@,1,MEDICARE,IDX +MEDICO INSURANCE CO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MEGA LIFE & HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MERITAIN HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +METRO CARE,Other,349,Other,EPIC +MH NET,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MISSOURI CARE,Medicaid Repl OOS,25,Medicaid - Out of State,EPIC +MO MEDICAID,Medicaid OOS,25,Medicaid - Out of State,EPIC +MO MEDICAID PENDING,MEDICAID PENDING,NI,No Information,EPIC +MOAA MEDIPLUS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MODEL MC PAYOR,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MULTIPLAN,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MUTUAL OF OMAHA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +NATIONAL ASSN LETTER CARRIERS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +NECA IBEW FAMILY MEDICAL PLAN,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +NORTH AMERICAN INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +OLD SURETY LIFE INSURANCE CO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +OPTUMHEALTH,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +ORSCHELN INDUSTRIES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +PHP,BCBS,6,BLUE CROSS/BLUE SHIELD,EPIC +PHYSICIANS MUTUAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +PLUMBERS LOCAL 8,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +PRINCIPAL FINANCIAL GROUP,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +PRISON HEALTH SYSTEMS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +PYRAMID LIFE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +QUIK TRIP,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +REI,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RESERVE NATIONAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX ADVANCE PCS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX AETNA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX ALLWIN DATA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX ALPHASCRIP,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX ARGUS HEALTH SYSTEMS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX CATALYST,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX CIGNA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX CVS/CAREMARK,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX EMDEON,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX EMPLOYEE BENEFITS MANAGEMENT SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX ENVISIONRX,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX EXPRESS SCRIPTS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX HEALTHTRANS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX HUMANA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX LDI INTEGRATED PHARMACY SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX MAGELLAN,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX MAXOR PLUS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX MEDCO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX MEDIMPACT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX MEDTRAK,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX MO MEDICAID,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX NATIONAL PRESCRIPTION SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX NETCARD SYSTEMS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX OPTUM RX,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX PBM PLUS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX PHARMACY DATA MANAGEMENT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX PRIME THERAPEUTICS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX PROCARE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX RESTAT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX RYAN WHITE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX SAV-RX,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX THERAPY FIRST,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX TMESYS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX UNITEDHEALTHCARE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX US SCRIPT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +SANFORD HEALTH PLAN,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +SEECHANGE HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +SELMAN AND COMPANY,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +SHEET METAL WORKERS NATIONAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STANDARD LIFE & ACCIDENT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STAR HRG/STAR ADMIN SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STARMARK,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STATE FARM INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STATE OF KANSAS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STATE OF MISSOURI,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STERLING LIFE INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STUDENT ASSURANCE SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +THREE RIVERS PROVIDER NETWORK,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +THRIVENT FOR LUTHERANS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +TODAY'S OPTIONS,Medicare Repl,1,MEDICARE,EPIC +TRANSAMERICA LIFE INS CO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +TRICARE,Tricare,3113,TRICARE Standard - Fee For Service,EPIC +TRICARE HMO,@,3111,TRICARE Prime—HMO,IDX +TRICARE PPO/TRAD,@,3112,TRICARE Extra—PPO,IDX +TRICARE UHC HMO A,@,3111,TRICARE Prime—HMO,IDX +TRICARE UHC HMO B,@,3111,TRICARE Prime—HMO,IDX +TRICARE UHC PPO A,@,3112,TRICARE Extra—PPO,IDX +TRICARE UHC PPO B,@,3112,TRICARE Extra—PPO,IDX +TRIWEST,VA,32,Department of Veterans Affairs,EPIC +UHC,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +UHC MEDICAID KS,Medicaid Repl KS,2,MEDICAID,EPIC +UHC MEDICAID MO,Medicaid Repl OOS,25,Medicaid - Out of State,EPIC +UHC MEDICARE,Medicare Repl,1,MEDICARE,EPIC +UHC COMMUNITY PLAN KS A,@,2,PRIVATE HEALTH INSURANCE,IDX +UHC COMMUNITY PLAN KS B,@,2,PRIVATE HEALTH INSURANCE,IDX +UNITED AMERICAN INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +UNITED HEALTH/METRA HMO/POS A,@,5,PRIVATE HEALTH INSURANCE,IDX +UNITED HEALTH/METRA HMO/POS B,@,5,PRIVATE HEALTH INSURANCE,IDX +UNITED HEALTH/METRA PPO/TRAD A,@,5,PRIVATE HEALTH INSURANCE,IDX +UNITED HEALTH/METRA PPO/TRAD B,@,5,PRIVATE HEALTH INSURANCE,IDX +UPREHS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +US MARSHALL SERVICE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +USA MANAGED CARE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +USAA LIFE INS CO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +VA,VA,32,Department of Veterans Affairs,EPIC +VETERANS ADMINISTRATION,VA,32,Department of Veterans Affairs,EPIC +VIA CHRISTI HOPE,Medicaid Repl KS,2,MEDICAID,EPIC +VISION,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +WILSON-MCSHANE CORP,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +WORKERS COMP,Worker's Comp,95,Worker's Compensation,EPIC +WORKMAN'S COMPENSATION,@,95,Worker's Compensation,IDX +WPPA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +WYJO CARE,Other,349,Other,EPIC +AUTO/LIABILITY,@,9,MISCELLANEOUS/OTHER,IDX +BAD DEBT,@,83,Refusal to Pay/Bad Debt,IDX +CHARITY/HARDSHIP,@,821,Charity,IDX +DISABILITY DETERMINATIONS,@,9,MISCELLANEOUS/OTHER,IDX +KUH EMPL SUPPLEMENTAL PLAN,@,9,MISCELLANEOUS/OTHER,IDX +KUPI ADMINISTRATIVE,@,9,MISCELLANEOUS/OTHER,IDX +KUPI COMPLIANCE,@,9,MISCELLANEOUS/OTHER,IDX +LIABILITY OTHER (NOT AUTO/WC),@,9,MISCELLANEOUS/OTHER,IDX +OTHER/COMMERCIAL APPEALS-KUPI,@,5,PRIVATE HEALTH INSURANCE,IDX +OTHER/COMMERCIAL APPEALS-PER SE,@,5,PRIVATE HEALTH INSURANCE,IDX +OTHER/COMMERCIAL HMO/POS A,@,5,PRIVATE HEALTH INSURANCE,IDX +OTHER/COMMERCIAL HMO/POS B,@,5,PRIVATE HEALTH INSURANCE,IDX +OTHER/COMMERCIAL PPO/TRAD A,@,5,PRIVATE HEALTH INSURANCE,IDX +OTHER/COMMERCIAL PPO/TRAD B,@,5,PRIVATE HEALTH INSURANCE,IDX +SELF PAY,@,81,Self-pay,IDX +Medicare Traditional/Indemnity,@,12,Medicare (Non-managed Care),UHC +Commercial/Private Preferred Provider Organization (PPO),@,512,Commercial Managed Care - PPO,UHC +Medicaid/Managed Care,@,21,Medicaid (Managed Care),UHC +Medicaid Traditional/Indemnity,@,22,Medicaid (Non-managed Care Plan),UHC +Commercial/Private Health Maintenance Organization (HMO),@,511,Commercial Managed Care - HMO,UHC +Self-Pay - Uninsured NOS,@,81,Self-pay,UHC +Medicare/Managed Care,@,11,Medicare (Managed Care),UHC +Commercial/Private Traditional/Indemnity,@,52,Private Health Insurance - Indemnity,UHC +Commercial/Private University Employees,@,5,PRIVATE HEALTH INSURANCE,UHC +Military Veterans Administration,@,3,OTHER GOVERNMENT (Federal/State/Local) (excluding Department of Corrections),UHC +Military Tri-Care,@,311,TRICARE (CHAMPUS),UHC +County Medically Indigent Services NOS,@,3,OTHER GOVERNMENT (Federal/State/Local) (excluding Department of Corrections),UHC +Commercial/Private Transplant Network,@,5,PRIVATE HEALTH INSURANCE,UHC +Auto Insurance Traditional/Indemnity,@,96,Auto Insurance (includes no fault),UHC +Workers Compensation NOS,@,95,Worker's Compensation,UHC +Workers Compensation Traditional/Indemnity,@,951,Worker's Comp HMO,UHC +Charity NOS,@,821,Charity,UHC +State Assisted Healthcare NOS,@,3,OTHER GOVERNMENT (Federal/State/Local) (excluding Department of Corrections),UHC +State Assisted Healthcare Prisoners,@,4,DEPARTMENTS OF CORRECTIONS,UHC +Commercial/Private Prisoners,@,5,PRIVATE HEALTH INSURANCE,UHC +Medicare NOS,@,1,MEDICARE,UHC +Medicaid Transplant Network,@,2,MEDICAID,UHC +Research NOS,@,823,Research/Clinical Trial,UHC +Self-Pay - Cash in Full NOS,@,81,Self-pay,UHC +Commercial/Private Point-of-Service (POS),@,513,Commercial Managed Care - POS,UHC +Commercial/Private NOS,@,3,OTHER GOVERNMENT (Federal/State/Local) (excluding Department of Corrections),UHC From dafd45e3c6a0e4c0822453583422a678dfdc5736 Mon Sep 17 00:00:00 2001 From: schandaka Date: Mon, 23 Sep 2019 09:10:48 -0500 Subject: [PATCH 495/507] Update obs_clin.sql removing hard coded references to the schema pcornet_cdm --- Oracle/obs_clin.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Oracle/obs_clin.sql b/Oracle/obs_clin.sql index 3d5c598..3748cb8 100644 --- a/Oracle/obs_clin.sql +++ b/Oracle/obs_clin.sql @@ -1,14 +1,14 @@ insert into cdm_status (task, start_time) select 'obs_clin', sysdate from dual / BEGIN -PMN_DROPSQL('DROP TABLE pcornet_cdm.obs_clin'); +PMN_DROPSQL('DROP TABLE obs_clin'); END; / BEGIN -PMN_DROPSQL('pcornet_cdm.cardiolabcomponents'); +PMN_DROPSQL('cardiolabcomponents'); END; -create table pcornet_cdm.cardiolabcomponents as +create table cardiolabcomponents as select distinct cor.component_id,ceap.proc_name from clarity.order_results cor left join clarity.order_proc cop on cop.order_proc_id=cor.order_proc_id left join clarity.clarity_eap ceap on cop.proc_id=ceap.proc_id @@ -62,11 +62,11 @@ select obs_clin_seq.nextval obsclinid ,lab.raw_result raw_obsclin_result ,' ' raw_obsclin_modifier ,' ' raw_obsclin_unit -from pcornet_cdm.lab_result_cm lab -left join pcornet_cdm.cardiolabcomponents card on substr(lab.raw_facility_code,18)=card.component_id; +from lab_result_cm lab +left join cardiolabcomponents card on substr(lab.raw_facility_code,18)=card.component_id; / -create index obs_clin_idx on pcornet_cdm.obs_clin (PATID, ENCOUNTERID) +create index obs_clin_idx on obs_clin (PATID, ENCOUNTERID) / BEGIN @@ -79,4 +79,4 @@ set end_time = sysdate, records = (select count(*) from obs_clin) where task = 'obs_clin' / -select records + 1 from cdm_status where task = 'obs_clin' \ No newline at end of file +select records + 1 from cdm_status where task = 'obs_clin' From dcffa1dc1740ed3676b0c95877484fc3a14d49ab Mon Sep 17 00:00:00 2001 From: schandaka Date: Mon, 23 Sep 2019 15:15:34 -0500 Subject: [PATCH 496/507] Added missing '/' --- Oracle/obs_clin.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Oracle/obs_clin.sql b/Oracle/obs_clin.sql index 3748cb8..d3cc535 100644 --- a/Oracle/obs_clin.sql +++ b/Oracle/obs_clin.sql @@ -7,15 +7,15 @@ END; BEGIN PMN_DROPSQL('cardiolabcomponents'); END; - +/ create table cardiolabcomponents as select distinct cor.component_id,ceap.proc_name from clarity.order_results cor left join clarity.order_proc cop on cop.order_proc_id=cor.order_proc_id left join clarity.clarity_eap ceap on cop.proc_id=ceap.proc_id where ceap.proc_name like '%ECHOCARDIOGRAM%'; - +/ create sequence obs_clin_seq cache 2000; - +/ CREATE TABLE obs_clin( OBSCLINID varchar(50) NOT NULL, PATID varchar(50) NOT NULL, @@ -38,7 +38,7 @@ CREATE TABLE obs_clin( RAW_OBSCLIN_MODIFIER varchar(50) NULL, RAW_OBSCLIN_UNIT varchar(50) NULL ); - +/ insert into obs_clin(obsclinid,patid,encounterid,obsclin_providerid,obsclin_date,obsclin_time,obsclin_type,obsclin_code,obsclin_result_qual, obsclin_result_text,obsclin_result_snomed,obsclin_result_num,obsclin_result_modifier,obsclin_result_unit,raw_obsclin_name, raw_obsclin_code,raw_obsclin_type,raw_obsclin_result,raw_obsclin_modifier,raw_obsclin_unit) From c77b4f219f8f276975f2fbffdc76bb046e465ffa Mon Sep 17 00:00:00 2001 From: schandaka Date: Mon, 23 Sep 2019 15:16:16 -0500 Subject: [PATCH 497/507] Added missing '/' --- Oracle/obs_gen.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Oracle/obs_gen.sql b/Oracle/obs_gen.sql index efad7d2..53fed45 100644 --- a/Oracle/obs_gen.sql +++ b/Oracle/obs_gen.sql @@ -7,7 +7,7 @@ END; / create sequence obs_gen_seq cache 2000; - +/ CREATE TABLE obs_gen( OBSGENID varchar(50) NOT NULL, PATID varchar(50) NOT NULL, @@ -30,15 +30,15 @@ CREATE TABLE obs_gen( RAW_OBSGEN_RESULT varchar(50) NULL, RAW_OBSGEN_UNIT varchar(50) NULL ) - +/ drop table pcornet_cdm.obsgen_naaccr; - +/ create table pcornet_cdm.obsgen_naaccr as select patient_num, encounter_num,provider_id,start_date,tval_char,nval_num,substr(concept_cd, instr(concept_cd, '|') + 1, instr(concept_cd, ':') - instr(concept_cd, '|') - 1) as code_value,concept_cd from &&i2b2_data_schema.observation_fact where concept_cd like '%NAACCR%'; - +/ insert into obs_gen(obsgenid,patid,encounterid,obsgen_providerid,obsgen_date,obsgen_code,obsgen_result_text, obsgen_result_num,obsgen_source,raw_obsgen_code) select @@ -55,7 +55,7 @@ obs.concept_cd raw_obsgen_code from pcornet_cdm.obsgen_naaccr obs left join pcornet_cdm.loinc_naaccr lc on obs.code_value =lc.code_value ; - +/ create index obs_gen_idx on obs_gen(PATID, ENCOUNTERID); / update cdm_status From 3f3f4edaef8f8cd426c30a0b903409384ba3e78c Mon Sep 17 00:00:00 2001 From: schandaka Date: Tue, 24 Sep 2019 15:17:41 -0500 Subject: [PATCH 498/507] Added missing ',' --- Oracle/encounter.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql index 82027a8..f322c84 100644 --- a/Oracle/encounter.sql +++ b/Oracle/encounter.sql @@ -134,7 +134,7 @@ left join payer_map pm on pm.payer_name = en.raw_payer_name_primary and (en.raw_ / create table encounter_w_fac_zip as select en.* -sf.tval_char facility_location +, sf.tval_char facility_location from encounter_w_fin en left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num and sf.source_column = 'FACILITY_ZIP' / From 6361037616255d1f9701e84fb0c7a3ef1e80a0f0 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 8 Apr 2020 15:38:23 -0500 Subject: [PATCH 499/507] script_lib: update tests w.r.t. med_admin.sql test is from 2018-02-21 fb605c9e med_admin.sql was updated 2018-05-10 b1ef59ca --- Oracle/med_admin.sql | 1 + script_lib.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Oracle/med_admin.sql b/Oracle/med_admin.sql index 3b97ada..e776601 100644 --- a/Oracle/med_admin.sql +++ b/Oracle/med_admin.sql @@ -155,3 +155,4 @@ set end_time = sysdate, records = (select count(*) from med_admin) where task = 'med_admin' / select records from cdm_status where task = 'med_admin' +/ diff --git a/script_lib.py b/script_lib.py index 1e6f8f4..c9f9937 100644 --- a/script_lib.py +++ b/script_lib.py @@ -18,9 +18,8 @@ >>> statements = Script.med_admin.statements() >>> print(next(s for s in statements if 'insert' in s)) ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE - create or replace trigger med_admin_trg - before insert on med_admin - ... + /** ... */ + insert into cdm_status (task, start_time) select 'med_admin', sysdate from dual A bit of sqlplus syntax is supported for ignoring errors in just part of a script: @@ -37,7 +36,7 @@ >>> print(statements[-1]) ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE - select 1 from cdm_status where status = 'med_admin' + select records from cdm_status where task = 'med_admin' The completion test may depend on a digest of the script and its dependencies: From 4d1a76d26833aa2ec9d9cb9ea7adb1f4855405a8 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 8 Apr 2020 15:48:48 -0500 Subject: [PATCH 500/507] lint: flake8 style check --- Oracle/backup_cdm.py | 4 ++-- csv_load.py | 3 ++- etl_tasks.py | 11 +++++++---- eventlog.py | 2 +- i2p_tasks.py | 6 ++++-- setup.cfg | 9 +++++++-- 6 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Oracle/backup_cdm.py b/Oracle/backup_cdm.py index 2d54e55..166cf1d 100644 --- a/Oracle/backup_cdm.py +++ b/Oracle/backup_cdm.py @@ -9,7 +9,7 @@ # Allow --dry run even without cx_Oracle try: from cx_Oracle import DatabaseError -except: +except ImportError: pass CDM_SPEC = '../2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv' # noqa @@ -60,7 +60,7 @@ def count_rows(cursor, schema_table): cursor.execute('select count(*) from %(schema_table)s' % dict(schema_table=schema_table)) ret = int(cursor.fetchall()[0][0]) - except: + except: # noqa pass return ret diff --git a/csv_load.py b/csv_load.py index 1b6e245..5379bc5 100644 --- a/csv_load.py +++ b/csv_load.py @@ -4,7 +4,7 @@ from param_val import StrParam from typing import Dict -from sqlalchemy import func, MetaData, Table, Column # type: ignore +from sqlalchemy import MetaData, Table, Column # type: ignore from sqlalchemy.types import String # type: ignore @@ -12,6 +12,7 @@ log = logging.getLogger(__name__) + class LoadCSV(CDMStatusTask): ''' Creates a table in the db with the name taskName. diff --git a/etl_tasks.py b/etl_tasks.py index c610978..626b759 100644 --- a/etl_tasks.py +++ b/etl_tasks.py @@ -388,7 +388,7 @@ def complete(self) -> bool: # If true, the task has not been logged in the CDM status table or has been logged and is in an # inconsistent state with the number of records set to null. - if statusTableRecordCount == None: + if statusTableRecordCount is None: return False log.info('task %s has %d rows', self.taskName, statusTableRecordCount) @@ -407,16 +407,19 @@ def setTaskEnd(self, rowCount: int) -> None: ''' Updates the taskName entry in the CDM status table with an end time of now and a count of records. ''' - statusTable = Table("cdm_status", MetaData(), Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) + statusTable = Table("cdm_status", MetaData(), + Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) db = self._dbtarget().engine - db.execute(statusTable.update().where(statusTable.c.TASK == self.taskName), [{'END_TIME': datetime.now(), 'RECORDS': rowCount}]) + db.execute(statusTable.update().where(statusTable.c.TASK == self.taskName), + [{'END_TIME': datetime.now(), 'RECORDS': rowCount}]) def setTaskStart(self) -> None: ''' Adds taskName to the CDM status table with a start time of now. ''' - statusTable = Table("cdm_status", MetaData(), Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) + statusTable = Table("cdm_status", MetaData(), + Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) db = self._dbtarget().engine db.execute(statusTable.insert(), [{'TASK': self.taskName, 'START_TIME': datetime.now()}]) diff --git a/eventlog.py b/eventlog.py index 30a6c12..3067cf3 100644 --- a/eventlog.py +++ b/eventlog.py @@ -111,7 +111,7 @@ def step(self, msg: str, argobj: Dict[str, object], outcome = logging.INFO try: yield LogState(msgparts, argobj, extra) - except: + except: # noqa outcome = logging.ERROR raise finally: diff --git a/i2p_tasks.py b/i2p_tasks.py index e9720db..87a8986 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -203,6 +203,7 @@ def results(self) -> List[RowProxy]: except DatabaseError: return [] + # TODO: pcornet_init drops and recreates the cdm_status table, forcing all tasks to wait until init is done. # Moving this operation to a distinct task would allow some other tasks (e.g. mapping tasks) to proceed, while init # performs other labor intensive SQL operations. @@ -285,13 +286,14 @@ class loadLabNormal(LoadCSV): def requires(self) -> List[luigi.Task]: return [pcornet_init()] - + + class loadLabRUnit(LoadCSV): taskName = 'LABRUNIT' csvname = 'curated_data/resultunit_manualcuration.csv' def requires(self) -> List[luigi.Task]: - return [pcornet_init()] + return [pcornet_init()] class loadHarvestLocal(LoadCSV): diff --git a/setup.cfg b/setup.cfg index 442184a..d78ceb4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,12 +11,17 @@ where=. [flake8] # E731: Assigning to lambda seems OK # E126: Emacs python mode seems to over-indent? -ignore = E731, E126 +# W605: TODO: good idea, but our code was written before the tool pointed this out. +# E252: TODO: community style seems to have changed after we wrote our code. +# W504: er.. you complain about line breaks on both sides. what are we supposed to do? +# E741: ambiguous variable name. TODO: good idea, but wasn't there when we wrote the code +ignore = E731, E126, W605, E252, W504, E741 + # with type annotations, 79 is awkward # guide, for window sizing: # 23456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 max-line-length = 120 -exclude = pythonjsonlogger,ADD_SCILHS_100 +exclude = pythonjsonlogger,ADD_SCILHS_100,.direnv [mypy] From b61dff4c49e42eb448c54abe34108bcc4e6116cb Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 8 Apr 2020 15:53:44 -0500 Subject: [PATCH 501/507] static typing lint (mypy 0.720) --- CONTRIBUTING.md | 2 +- csv_load.py | 2 +- etl_tasks.py | 8 ++++---- script_lib.py | 2 +- setup.cfg | 1 - stubs/cx_Oracle.pyi | 3 --- 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a8e9f6b..cf04b71 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -99,7 +99,7 @@ pass tests, style checks, and static type checking: $ nosetests && flake8 . && mypy . -_tested with mypy-0.521_ +_tested with mypy-0.720_ ### Checking in emacs diff --git a/csv_load.py b/csv_load.py index 5379bc5..2b5438c 100644 --- a/csv_load.py +++ b/csv_load.py @@ -4,7 +4,7 @@ from param_val import StrParam from typing import Dict -from sqlalchemy import MetaData, Table, Column # type: ignore +from sqlalchemy import MetaData, Table, Column from sqlalchemy.types import String # type: ignore diff --git a/etl_tasks.py b/etl_tasks.py index 626b759..73e1a81 100644 --- a/etl_tasks.py +++ b/etl_tasks.py @@ -12,7 +12,7 @@ import logging from luigi.contrib.sqla import SQLAlchemyTarget -from sqlalchemy import text as sql_text, Column, func, MetaData, Table # type: ignore +from sqlalchemy import text as sql_text, Column, func, MetaData, Table from sqlalchemy.engine import Connection, Engine from sqlalchemy.engine.result import ResultProxy from sqlalchemy.engine.url import make_url @@ -401,7 +401,7 @@ def getRecordCountFromTable(self) -> int: # This op is out of sync with the rest of the class, in that it assumes # the task must represent the creation of a table in the db. with self.connection() as q: - return q.scalar(sqla.select([func.count()]).select_from(self.taskName)) + return q.scalar(sqla.select([func.count()]).select_from(self.taskName)) # type: ignore def setTaskEnd(self, rowCount: int) -> None: ''' @@ -447,7 +447,7 @@ def explain_plan(work: LoggedConnection, statement: SQL) -> List[str]: # https://docs.oracle.com/cd/B19306_01/server.102/b14211/ex_plan.htm plan = work.execute( 'SELECT PLAN_TABLE_OUTPUT line FROM TABLE(DBMS_XPLAN.DISPLAY())') - return [row.line for row in plan] # type: ignore # sqla + return [row.line for row in plan] def maybe_ora_err(exc: Exception) -> Opt[Ora_Error]: @@ -759,7 +759,7 @@ def tryConnect(cls, engine: Engine) -> Connection: raise ConnectionProblem.refine(exc, str(engine)) from None @classmethod - def refine(cls, exc: Exception, conn_label: str) -> Exception: + def refine(cls, exc: DatabaseError, conn_label: str) -> Exception: '''Recognize known connection problems. :returns: customized exception for known diff --git a/script_lib.py b/script_lib.py index c9f9937..cfdff30 100644 --- a/script_lib.py +++ b/script_lib.py @@ -187,7 +187,7 @@ def _get_deps(cls, sql: Text) -> List['SQLMixin']: if not m: return [] name, ext = m.group(1).rsplit('.', 1) - choices = Script if ext == 'sql' else [] + choices = list(Script) if ext == 'sql' else [] deps = [cast(SQLMixin, s) for s in choices if s.name == name] if not deps: raise KeyError(name) diff --git a/setup.cfg b/setup.cfg index d78ceb4..493067d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,6 @@ warn_redundant_casts=true warn_unused_ignores=true strict_optional=true -strict_boolean=false # but all the other --strict flags (from mypy -h): disallow_untyped_calls=true disallow_untyped_defs=true diff --git a/stubs/cx_Oracle.pyi b/stubs/cx_Oracle.pyi index 10a8948..f96e852 100644 --- a/stubs/cx_Oracle.pyi +++ b/stubs/cx_Oracle.pyi @@ -337,11 +337,8 @@ class TIMESTAMP(_BASEVARTYPE): ... class Timestamp(date): hour = ... # type: Any - max = ... # type: Any microsecond = ... # type: Any - min = ... # type: Any minute = ... # type: Any - resolution = ... # type: Any second = ... # type: Any tzinfo = ... # type: Any def __init__(self, *args, **kwargs): ... From e6083143accba4c0bdb38900381eca02166c540a Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 8 Apr 2020 16:44:27 -0500 Subject: [PATCH 502/507] CDMStatusTask.setTaskStart: prune failed attempts ... to avoid ORA-00001: unique constraint --- etl_tasks.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/etl_tasks.py b/etl_tasks.py index 73e1a81..b040196 100644 --- a/etl_tasks.py +++ b/etl_tasks.py @@ -379,6 +379,9 @@ class CDMStatusTask(DBAccessTask): # Basic status check, assume the typical task produces at least one record. expectedRecords = IntParam(default=1) + statusTable = Table("cdm_status", MetaData(), + Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) + def complete(self) -> bool: ''' Complete when the CDM status table reports at least as many records as expected for the task. @@ -407,22 +410,21 @@ def setTaskEnd(self, rowCount: int) -> None: ''' Updates the taskName entry in the CDM status table with an end time of now and a count of records. ''' - statusTable = Table("cdm_status", MetaData(), - Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) - + st = self.statusTable db = self._dbtarget().engine - db.execute(statusTable.update().where(statusTable.c.TASK == self.taskName), + db.execute(st.update().where(st.c.TASK == self.taskName), [{'END_TIME': datetime.now(), 'RECORDS': rowCount}]) def setTaskStart(self) -> None: ''' Adds taskName to the CDM status table with a start time of now. ''' - statusTable = Table("cdm_status", MetaData(), - Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) - + st = self.statusTable db = self._dbtarget().engine - db.execute(statusTable.insert(), [{'TASK': self.taskName, 'START_TIME': datetime.now()}]) + # prune any failed attempt + db.execute(st.delete().where(sqla.and_(st.c.TASK == self.taskName, + st.c.END_TIME == None))) # noqa + db.execute(st.insert(), [{'TASK': self.taskName, 'START_TIME': datetime.now()}]) def log_plan(lc: LoggedConnection, event: str, params: Dict[str, Any], From 4dd4baca4255a22d511460cbf4814463ba5e0796 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 8 Apr 2020 16:45:31 -0500 Subject: [PATCH 503/507] curated harvest_local: prune empty column SQLAlchemy got more strict? DictReader changed? ``` sqlalchemy.exc.ArgumentError: Column must be constructed with a non-blank name or assign a non-blank .name before adding to a Table. ``` --- curated_data/harvest_local.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/curated_data/harvest_local.csv b/curated_data/harvest_local.csv index db02154..4f16835 100644 --- a/curated_data/harvest_local.csv +++ b/curated_data/harvest_local.csv @@ -1,2 +1,2 @@ -DATAMART_CLAIMS,DATAMART_EHR,BIRTH_DATE_MGMT,ENR_START_DATE_MGMT,ENR_END_DATE_MGMT,ADMIT_DATE_MGMT,DISCHARGE_DATE_MGMT,DX_DATE_MGMT,PX_DATE_MGMT,RX_ORDER_DATE_MGMT,RX_START_DATE_MGMT,RX_END_DATE_MGMT,DISPENSE_DATE_MGMT,LAB_ORDER_DATE_MGMT,SPECIMEN_DATE_MGMT,RESULT_DATE_MGMT,MEASURE_DATE_MGMT,ONSET_DATE_MGMT,REPORT_DATE_MGMT,RESOLVE_DATE_MGMT,PRO_DATE_MGMT,DEATH_DATE_MGMT,MEDADMIN_START_DATE_MGMT,MEDADMIN_STOP_DATE_MGMT,OBSCLIN_DATE_MGMT,OBSGEN_DATE_MGMT,ADDRESS_PERIOD_START_MGMT,ADDRESS_PERIOD_END_MGMT,VX_RECORD_DATE_MGMT,VX_ADMIN_DATE_MGMT,VX_EXP_DATE_MGMT, -1,2,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01, +DATAMART_CLAIMS,DATAMART_EHR,BIRTH_DATE_MGMT,ENR_START_DATE_MGMT,ENR_END_DATE_MGMT,ADMIT_DATE_MGMT,DISCHARGE_DATE_MGMT,DX_DATE_MGMT,PX_DATE_MGMT,RX_ORDER_DATE_MGMT,RX_START_DATE_MGMT,RX_END_DATE_MGMT,DISPENSE_DATE_MGMT,LAB_ORDER_DATE_MGMT,SPECIMEN_DATE_MGMT,RESULT_DATE_MGMT,MEASURE_DATE_MGMT,ONSET_DATE_MGMT,REPORT_DATE_MGMT,RESOLVE_DATE_MGMT,PRO_DATE_MGMT,DEATH_DATE_MGMT,MEDADMIN_START_DATE_MGMT,MEDADMIN_STOP_DATE_MGMT,OBSCLIN_DATE_MGMT,OBSGEN_DATE_MGMT,ADDRESS_PERIOD_START_MGMT,ADDRESS_PERIOD_END_MGMT,VX_RECORD_DATE_MGMT,VX_ADMIN_DATE_MGMT,VX_EXP_DATE_MGMT +1,2,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01 From 105a031b60a2b8e3e9d82062d23dd86053431bb6 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 8 Apr 2020 17:41:19 -0500 Subject: [PATCH 504/507] payer_map: convert to utf-8 --- curated_data/payer_map.csv | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/curated_data/payer_map.csv b/curated_data/payer_map.csv index 7364238..15f8c13 100644 --- a/curated_data/payer_map.csv +++ b/curated_data/payer_map.csv @@ -248,12 +248,12 @@ THRIVENT FOR LUTHERANS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC TODAY'S OPTIONS,Medicare Repl,1,MEDICARE,EPIC TRANSAMERICA LIFE INS CO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC TRICARE,Tricare,3113,TRICARE Standard - Fee For Service,EPIC -TRICARE HMO,@,3111,TRICARE Prime—HMO,IDX -TRICARE PPO/TRAD,@,3112,TRICARE Extra—PPO,IDX -TRICARE UHC HMO A,@,3111,TRICARE Prime—HMO,IDX -TRICARE UHC HMO B,@,3111,TRICARE Prime—HMO,IDX -TRICARE UHC PPO A,@,3112,TRICARE Extra—PPO,IDX -TRICARE UHC PPO B,@,3112,TRICARE Extra—PPO,IDX +TRICARE HMO,@,3111,TRICARE Prime—HMO,IDX +TRICARE PPO/TRAD,@,3112,TRICARE Extra—PPO,IDX +TRICARE UHC HMO A,@,3111,TRICARE Prime—HMO,IDX +TRICARE UHC HMO B,@,3111,TRICARE Prime—HMO,IDX +TRICARE UHC PPO A,@,3112,TRICARE Extra—PPO,IDX +TRICARE UHC PPO B,@,3112,TRICARE Extra—PPO,IDX TRIWEST,VA,32,Department of Veterans Affairs,EPIC UHC,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC UHC MEDICAID KS,Medicaid Repl KS,2,MEDICAID,EPIC From a1fd9deee02adf01bb9b4b5571319614873590ec Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 8 Apr 2020 17:11:37 -0500 Subject: [PATCH 505/507] obs_gen, obs_clin, ...: use lights-out style - leave `insert into cdm_status` to the task framework - separate with / only, not both ; and / - drop before creating --- Oracle/condition.sql | 2 -- Oracle/diagnosis.sql | 4 +--- Oracle/obs_clin.sql | 24 +++++++++++++----------- Oracle/obs_gen.sql | 25 +++++++++++++++---------- 4 files changed, 29 insertions(+), 26 deletions(-) diff --git a/Oracle/condition.sql b/Oracle/condition.sql index 8ca921e..f281899 100644 --- a/Oracle/condition.sql +++ b/Oracle/condition.sql @@ -1,7 +1,5 @@ /** condition - create and populate the condition table. */ -insert into cdm_status (task, start_time) select 'condition', sysdate from dual -/ BEGIN PMN_DROPSQL('DROP TABLE condition'); END; diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql index cef6c09..166767d 100644 --- a/Oracle/diagnosis.sql +++ b/Oracle/diagnosis.sql @@ -1,7 +1,5 @@ /** diagnosis - create and populate the diagnosis table. */ -insert into cdm_status (task, start_time) select 'diagnosis', sysdate from dual -/ BEGIN PMN_DROPSQL('DROP TABLE diagnosis'); END; @@ -309,4 +307,4 @@ update cdm_status set end_time = sysdate, records = (select count(*) from diagnosis) where task = 'diagnosis' / -select records from cdm_status where task = 'diagnosis'; \ No newline at end of file +select records from cdm_status where task = 'diagnosis' \ No newline at end of file diff --git a/Oracle/obs_clin.sql b/Oracle/obs_clin.sql index d3cc535..44f229c 100644 --- a/Oracle/obs_clin.sql +++ b/Oracle/obs_clin.sql @@ -1,20 +1,22 @@ -insert into cdm_status (task, start_time) select 'obs_clin', sysdate from dual -/ -BEGIN -PMN_DROPSQL('DROP TABLE obs_clin'); -END; -/ BEGIN -PMN_DROPSQL('cardiolabcomponents'); +PMN_DROPSQL('drop table cardiolabcomponents'); END; / create table cardiolabcomponents as select distinct cor.component_id,ceap.proc_name from clarity.order_results cor left join clarity.order_proc cop on cop.order_proc_id=cor.order_proc_id left join clarity.clarity_eap ceap on cop.proc_id=ceap.proc_id -where ceap.proc_name like '%ECHOCARDIOGRAM%'; +where ceap.proc_name like '%ECHOCARDIOGRAM%' +/ +BEGIN +PMN_DROPSQL('DROP SEQUENCE obs_clin_seq'); +END; +/ +create sequence obs_clin_seq cache 2000 / -create sequence obs_clin_seq cache 2000; +BEGIN +PMN_DROPSQL('DROP TABLE obs_clin'); +END; / CREATE TABLE obs_clin( OBSCLINID varchar(50) NOT NULL, @@ -37,7 +39,7 @@ CREATE TABLE obs_clin( RAW_OBSCLIN_RESULT varchar(50) NULL, RAW_OBSCLIN_MODIFIER varchar(50) NULL, RAW_OBSCLIN_UNIT varchar(50) NULL -); +) / insert into obs_clin(obsclinid,patid,encounterid,obsclin_providerid,obsclin_date,obsclin_time,obsclin_type,obsclin_code,obsclin_result_qual, obsclin_result_text,obsclin_result_snomed,obsclin_result_num,obsclin_result_modifier,obsclin_result_unit,raw_obsclin_name, @@ -63,7 +65,7 @@ select obs_clin_seq.nextval obsclinid ,' ' raw_obsclin_modifier ,' ' raw_obsclin_unit from lab_result_cm lab -left join cardiolabcomponents card on substr(lab.raw_facility_code,18)=card.component_id; +left join cardiolabcomponents card on substr(lab.raw_facility_code,18)=card.component_id / create index obs_clin_idx on obs_clin (PATID, ENCOUNTERID) diff --git a/Oracle/obs_gen.sql b/Oracle/obs_gen.sql index 53fed45..ec5a8d4 100644 --- a/Oracle/obs_gen.sql +++ b/Oracle/obs_gen.sql @@ -1,13 +1,15 @@ /* obs_gen - create the obs_gen table.*/ -insert into cdm_status (task, start_time) select 'obs_gen', sysdate from dual + +BEGIN +PMN_DROPSQL('drop sequence obs_gen_seq'); +END; +/ +create sequence obs_gen_seq cache 2000 / BEGIN PMN_DROPSQL('DROP TABLE obs_gen'); END; / - -create sequence obs_gen_seq cache 2000; -/ CREATE TABLE obs_gen( OBSGENID varchar(50) NOT NULL, PATID varchar(50) NOT NULL, @@ -32,13 +34,16 @@ CREATE TABLE obs_gen( ) / -drop table pcornet_cdm.obsgen_naaccr; +BEGIN +PMN_DROPSQL('drop table pcornet_cdm.obsgen_naaccr'); +END; / create table pcornet_cdm.obsgen_naaccr as select patient_num, encounter_num,provider_id,start_date,tval_char,nval_num,substr(concept_cd, instr(concept_cd, '|') + 1, instr(concept_cd, ':') - instr(concept_cd, '|') - 1) as code_value,concept_cd from &&i2b2_data_schema.observation_fact - where concept_cd like '%NAACCR%'; -/ + where concept_cd like '%NAACCR%' +/ + insert into obs_gen(obsgenid,patid,encounterid,obsgen_providerid,obsgen_date,obsgen_code,obsgen_result_text, obsgen_result_num,obsgen_source,raw_obsgen_code) select @@ -54,13 +59,13 @@ obs.nval_num obsgen_result_num, obs.concept_cd raw_obsgen_code from pcornet_cdm.obsgen_naaccr obs left join pcornet_cdm.loinc_naaccr lc on obs.code_value =lc.code_value -; / -create index obs_gen_idx on obs_gen(PATID, ENCOUNTERID); +create index obs_gen_idx on obs_gen(PATID, ENCOUNTERID) / update cdm_status set end_time = sysdate, records = (select count(*) from obs_gen) where task = 'obs_gen' / -select records + 1 from cdm_status where task = 'obs_gen'; +select records + 1 from cdm_status where task = 'obs_gen' +/ From d1796482fd11c39be64f133e3c56393c0da3ca69 Mon Sep 17 00:00:00 2001 From: Nathan Hensel Date: Wed, 3 Jun 2020 11:25:10 -0500 Subject: [PATCH 506/507] add pcornet_init dependency to pcornet_loader to clear cdm_status --- i2p_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/i2p_tasks.py b/i2p_tasks.py index 87a8986..56b4cbe 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -222,7 +222,7 @@ class pcornet_loader(I2PScriptTask): script = Script.pcornet_loader def requires(self) -> List[luigi.Task]: - return [harvest()] + return [pcornet_init(), harvest()] class pcornet_trial(I2PScriptTask): From 4299f2789314c7fe5f49a4e0917168d4df5ff9ce Mon Sep 17 00:00:00 2001 From: Sravani Date: Thu, 9 Jul 2020 10:18:42 -0500 Subject: [PATCH 507/507] Added dispense_source column to the dispensing table --- Oracle/dispensing.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql index 893a286..c66f5dc 100644 --- a/Oracle/dispensing.sql +++ b/Oracle/dispensing.sql @@ -20,7 +20,8 @@ CREATE TABLE dispensing( RAW_NDC varchar (50) NULL, RAW_DISPENSE_DOSE_DISP varchar(50) NULL, RAW_DISPENSE_DOSE_DISP_UNIT varchar(50) NULL, - RAW_DISPENSE_ROUTE varchar(50) NULL + RAW_DISPENSE_ROUTE varchar(50) NULL, + DISPENSE_SOURCE varchar(2) NULL ) / @@ -104,6 +105,7 @@ insert into dispensing ( ,RAW_DISPENSE_DOSE_DISP ,RAW_DISPENSE_DOSE_DISP_UNIT ,RAW_DISPENSE_ROUTE + ,DISPENSE_SOURCE ) /* Below is the Cycle 2 fix for populating the DISPENSING table */ with disp_status as ( @@ -168,7 +170,8 @@ select distinct null raw_ndc, dd.dose raw_dispense_dose_disp, du.tval_char raw_dispense_dose_disp_unit, - dr.tval_char raw_dispense_route + dr.tval_char raw_dispense_route, + 'PM' dispense_source from disp_status st left outer join disp_quantity qt on st.patient_num=qt.patient_num