diff --git a/.gitignore b/.gitignore index 872921f..0737f94 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.exe *.o *.so +*.pyc # Temporary files # ################### diff --git a/2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv b/2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv new file mode 100644 index 0000000..6b0fc84 --- /dev/null +++ b/2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv @@ -0,0 +1,958 @@ +TABLE_NAME,FIELD_NAME,RDBMS_DATA_TYPE,SAS_DATA_TYPE,DATA_FORMAT,REPLICATED,UNIT_OF_MEASURE,VALUE_SET,VALUE_DESCRIPTION,DEFINITION,CDM_ORDER +DEMOGRAPHIC,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary person-level identifier used to link across tables. + +PATID is a pseudoidentifier with a consistent crosswalk to the true identifier retained by the source Data Partner. For analytical data sets requiring patient-level data, only the pseudoidentifier is used to link across all information belonging to a patient. + +The PATID must be unique within the data source being queried. Creating a unique identifier within a CDRN would be beneficial and acceptable. The PATID is not the basis for linkages across partners.",1 +DEMOGRAPHIC,BIRTH_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date of birth.,2 +DEMOGRAPHIC,BIRTH_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Time of birth.,3 +DEMOGRAPHIC,SEX,RDBMS Text(2),SAS Char(2),,NO,,A;F;M;NI;UN;OT,"A = Ambiguous +F = Female +M = Male +NI = No information +UN = Unknown +OT = Other","Administrative sex. + +v2.0 guidance added: The ?Ambiguous? category may be used for individuals who are physically undifferentiated from birth. The ?Other? category may be used for individuals who are undergoing gender re-assignment.",4 +DEMOGRAPHIC,HISPANIC,RDBMS Text(2),SAS Char(2),,NO,,Y;N;R;NI;UN;OT,"Y = Yes +N = No +R = Refuse to answer +NI = No information +UN = Unknown +OT = Other","A person of Cuban, Mexican, Puerto Rican, South or Central American, or other Spanish culture or origin, regardless of race. + +v2.0 amendment: The new categorical value of ?Refuse to answer? has been added.",5 +DEMOGRAPHIC,RACE,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;05;06;07;NI;UN;OT,"01 = American Indian or Alaska Native +02 = Asian +03 = Black or African American +04 = Native Hawaiian or Other Pacific Islander +05 = White +06 = Multiple race +07 = Refuse to answer +NI = No information +UN = Unknown +OT = Other","Please use only one race value per patient. + +Details of categorical definitions: +American Indian or Alaska Native: A person having origins in any of the original peoples of North and South America (including Central America), and who maintains tribal affiliation or community attachment. + +Asian: A person having origins in any of the original peoples of the Far East, Southeast Asia, or the Indian subcontinent including, for example, Cambodia, China, India, Japan, Korea, Malaysia, Pakistan, the Philippine Islands, Thailand, and Vietnam. + +Black or African American: A person having origins in any of the black racial groups of Africa. + +Native Hawaiian or Other Pacific Islander: A person having origins in any of the original peoples of Hawaii, Guam, Samoa, or other Pacific Islands. + +White: A person having origins in any of the original peoples of Europe, the Middle East, or North Africa.",6 +DEMOGRAPHIC,BIOBANK_FLAG,RDBMS Text(1),SAS Char(1),,NO,,Y;N,"Y = Yes +N = No","Flag to indicate that one or more biobanked specimens are stored and available for research use. Examples of biospecimens could include plasma, urine, or tissue. If biospecimens are available, locally maintained ?mapping tables? would be necessary to map between the DEMOGRAPHIC record and the originating biobanking system(s). + +If no known biobanked specimens are available, this field should be marked ?No?.",7 +DEMOGRAPHIC,RAW_SEX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",8 +DEMOGRAPHIC,RAW_HISPANIC,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",9 +DEMOGRAPHIC,RAW_RACE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",10 +ENROLLMENT,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables.,11 +ENROLLMENT,ENR_START_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date of the beginning of the enrollment period. If the exact date is unknown, use the first day of the month. + +For implementation of the CDM, a long span of longitudinal data is desirable; however, especially for historical data more than a decade old, the appropriate beginning date should be determined by the partner?s knowledge of the validity and usability of the data. More specific guidance can be provided through implementation discussions.",12 +ENROLLMENT,ENR_END_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date of the end of the enrollment period. If the exact date is unknown, use the last day of the month.",13 +ENROLLMENT,CHART,RDBMS Text(1),SAS Char(1),,NO,,Y;N,"Y = Yes +N = No","Chart abstraction flag is intended to answer the question, ""Are you able to request (or review) charts for this person?"" This flag does not address chart availability. Mark as ""Yes"" if there are no contractual or other restrictions between you and the individual (or sponsor) that would prohibit you from requesting any chart for this member. + +Note: This field is most relevant for health insurers that can request charts from affiliated providers. This field allows exclusion of patients from studies that require chart review to validate exposures and/or outcomes. It identifies patients for whom charts are never available and for whom the chart can never be requested.",14 +ENROLLMENT,ENR_BASIS,RDBMS Text(1),SAS Char(1),,NO,,I;G;A;E,"I = Insurance +G = Geography +A = Algorithmic +E = Encounter-based","When insurance information is not available but complete capture can be asserted some other way, please identify the basis on which complete capture is defined. Additional information on the approach identified will be required from each partner. + +ENR_BASIS is a property of the time period defined. A patient can have multiple entries in the table. + +Details of categorical definitions: +Insurance: The start and stop dates are based upon the concept of enrollment with health plan insurance + +Geography: An assertion of complete data capture between the start and end dates based upon geographic characteristics, such as regional isolation + +Algorithmic: An assertion of complete data capture between the start and end dates, based on a locally developed or applied algorithm, often using multiple criteria + +Encounter-based: The start and stop dates are populated from the earliest-observed encounter and latest-observed encounter.",15 +ENCOUNTER,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier. Used to link across tables, including the ENCOUNTER, DIAGNOSIS, and PROCEDURE tables.",16 +ENCOUNTER,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables.,17 +ENCOUNTER,ADMIT_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Encounter or admission date.,18 +ENCOUNTER,ADMIT_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Encounter or admission time.,19 +ENCOUNTER,DISCHARGE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Discharge date. Should be populated for all Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),20 +ENCOUNTER,DISCHARGE_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Discharge time.,21 +ENCOUNTER,PROVIDERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Provider code for the provider who is most responsible for this encounter. For encounters with multiple providers choose one so the encounter can be linked to the diagnosis and procedure tables. As with the PATID, the provider code is a pseudoidentifier with a consistent crosswalk to the real identifier.",22 +ENCOUNTER,FACILITY_LOCATION,RDBMS Text(3),SAS Char(3),,NO,,,,Geographic location (3 digit zip code). Should be null if not recorded in source system (modification made from ?blank? to ?null? in v3.0).,23 +ENCOUNTER,ENC_TYPE,RDBMS Text(2),SAS Char(2),,NO,,AV;ED;EI;IP;IS;OA;NI;UN;OT,"AV = Ambulatory Visit +ED = Emergency Department +EI = Emergency Department Admit to Inpatient Hospital Stay (permissible substitution) +IP = Inpatient Hospital Stay +IS = Non-Acute Institutional Stay +OA = Other Ambulatory Visit +NI = No information +UN = Unknown +OT = Other","Encounter type. + +Details of categorical definitions: +Ambulatory Visit: Includes visits at outpatient clinics, physician offices, same day/ambulatory surgery centers, urgent care facilities, and other same-day ambulatory hospital encounters, but excludes emergency department encounters. + +Emergency Department (ED): Includes ED encounters that become inpatient stays (in which case inpatient stays would be a separate encounter). Excludes urgent care visits. ED claims should be pulled before hospitalization claims to ensure that ED with subsequent admission won't be rolled up in the hospital event. + +Emergency Department Admit to Inpatient Hospital Stay: Permissible substitution for preferred state of separate ED and IP records. Only for use with data sources where the individual records for ED and IP cannot be distinguished (new to v2.0). + +Inpatient Hospital Stay: Includes all inpatient stays, including: same-day hospital discharges, hospital transfers, and acute hospital care where the discharge is after the admission date. + +Non-Acute Institutional Stay: Includes hospice, skilled nursing facility (SNF), rehab center, nursing home, residential, overnight non-hospital dialysis and other non-hospital stays. + +Other Ambulatory Visit: Includes other non-overnight AV encounters such as hospice visits, home health visits, skilled nursing facility visits, other non-hospital visits, as well as telemedicine, telephone and email consultations. May also include ""lab only"" visits (when a lab is ordered outside of a patient visit), ""pharmacy only"" (e.g., when a patient has a refill ordered without a face-to-face visit), ""imaging only"", etc.",24 +ENCOUNTER,FACILITYID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary local facility code that identifies the hospital or clinic. Used for chart abstraction and validation. + +FACILITYID can be a true identifier, or a pseudoidentifier with a consistent crosswalk to the true identifier retained by the source Data Partner.",25 +ENCOUNTER,DISCHARGE_DISPOSITION,RDBMS Text(2),SAS Char(2),,NO,,A;E;NI;UN;OT,"A = Discharged alive +E = Expired +NI = No information +UN = Unknown +OT = Other",Vital status at discharge. Should be populated for Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),26 +ENCOUNTER,DISCHARGE_STATUS,RDBMS Text(2),SAS Char(2),,NO,,AF;AL;AM;AW;EX;HH;HO;HS;IP;NH;RH;RS;SH;SN;NI;UN;OT,"AF = Adult Foster Home +AL = Assisted Living Facility +AM = Against Medical Advice +AW = Absent without leave +EX = Expired +HH = Home Health +HO = Home / Self Care +HS = Hospice +IP = Other Acute Inpatient Hospital +NH = Nursing Home (Includes ICF) +RH = Rehabilitation Facility +RS = Residential Facility +SH = Still In Hospital +SN = Skilled Nursing Facility +NI = No information +UN = Unknown +OT = Other",Discharge status. Should be populated for Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),27 +ENCOUNTER,DRG,RDBMS Text(3),SAS Char(3),,NO,,,,"3-digit Diagnosis Related Group (DRG). Should be populated for IP and IS encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for AV or OA encounters. Use leading zeroes for codes less than 100. (Additional guidance added in v3.0 for the EI encounter type.) + +The DRG is used for reimbursement for inpatient encounters. It is a Medicare requirement that combines diagnoses into clinical concepts for billing. Frequently used in observational data analyses. +",28 +ENCOUNTER,DRG_TYPE,RDBMS Text(2),SAS Char(2),,NO,,01;02;NI;UN;OT,"01 = CMS-DRG (old system) +02 = MS-DRG (current system) +NI = No information +UN = Unknown +OT = Other",DRG code version. MS-DRG (current system) began on 10/1/2007. Should be populated for IP and IS encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for AV or OA encounters. (Additional guidance added in v3.0 for the EI encounter type.),29 +ENCOUNTER,ADMITTING_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,AF;AL;AV;ED;HH;HO;HS;IP;NH;RH;RS;SN;NI;UN;OT,"AF = Adult Foster Home +AL = Assisted Living Facility +AV = Ambulatory Visit +ED = Emergency Department +HH = Home Health +HO = Home / Self Care +HS = Hospice +IP = Other Acute Inpatient Hospital +NH = Nursing Home (Includes ICF) +RH = Rehabilitation Facility +RS = Residential Facility +SN = Skilled Nursing Facility +NI = No information +UN = Unknown +OT = Other",Admitting source. Should be populated for Inpatient Hospital Stay (IP) and Non-Acute Institutional Stay (IS) encounter types. May be populated for Emergency Department (ED) and ED-to-Inpatient (EI) encounter types. Should be missing for ambulatory visit (AV or OA) encounter types. (Additional guidance added in v3.0 for the EI encounter type.),30 +ENCOUNTER,RAW_SITEID,RDBMS Text(x),SAS Char(x),,NO,,,,"This field is new to v2.0. + +Optional field for locally-defined identifier intended for local use; for example, where a network may have multiple sites contributing to a central data repository. + +This attribute may be sensitive in certain contexts; the intent is for internal network use only, and not to enable site quality comparisons.",31 +ENCOUNTER,RAW_ENC_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",32 +ENCOUNTER,RAW_DISCHARGE_DISPOSITION,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",33 +ENCOUNTER,RAW_DISCHARGE_STATUS,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",34 +ENCOUNTER,RAW_DRG_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",35 +ENCOUNTER,RAW_ADMITTING_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",36 +DIAGNOSIS,DIAGNOSISID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",37 +DIAGNOSIS,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,38 +DIAGNOSIS,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. Used to link across tables.,39 +DIAGNOSIS,ENC_TYPE,RDBMS Text(2),SAS Char(2),,YES,,AV;ED;EI;IP;IS;OA;NI;UN;OT,"AV = Ambulatory Visit +ED = Emergency Department +EI=Emergency Department Admit to Inpatient Hospital Stay (permissible substitution) +IP = Inpatient Hospital Stay +IS = Non-Acute Institutional Stay +OA = Other Ambulatory Visit +NI = No information +UN = Unknown +OT = Other",Please note: This is a field replicated from the ENCOUNTER table. See the ENCOUNTER table for definitions. ,40 +DIAGNOSIS,ADMIT_DATE,RDBMS Date,SAS Date (Numeric),,YES,DATE,,,Please note: This is a field replicated from the ENCOUNTER table. See the ENCOUNTER table for definitions. ,41 +DIAGNOSIS,PROVIDERID,RDBMS Text(x),SAS Char(x),,YES,,,,Please note: This is a field replicated from the ENCOUNTER table. See the ENCOUNTER table for definitions. ,42 +DIAGNOSIS,DX,RDBMS Text(18),SAS Char(18),,NO,,,,"Diagnosis code. + +Leading zeroes and different levels of decimal precision are permissible in this field. Please populate the exact textual value of this diagnosis code, but remove source-specific suffixes and prefixes. Other codes should be listed as recorded in the source data.",43 +DIAGNOSIS,DX_TYPE,RDBMS Text(2),SAS Char(2),,NO,,09;10;11;SM;NI;UN;OT,"09 = ICD-9-CM +10 = ICD-10-CM +11 = ICD-11-CM +SM = SNOMED CT +NI = No information +UN = Unknown +OT = Other","Diagnosis code type. + +We provide values for ICD and SNOMED code types. Other code types will be added as new terminologies are more widely used. + +Please note: The ?Other? category is meant to identify internal use ontologies and codes.",44 +DIAGNOSIS,DX_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,AD;DI;FI;IN;NI;UN;OT,"AD = Admitting +DI = Discharge +FI = Final +IN = Interim +NI = No information +UN = Unknown +OT = Other","Classification of diagnosis source. We include these categories to allow some flexibility in implementation. The context is to capture available diagnoses recorded during a specific encounter. It is not necessary to populate interim diagnoses unless readily available. + +Ambulatory encounters would generally be expected to have a source of ?Final.?",45 +DIAGNOSIS,PDX,RDBMS Text(2),SAS Char(2),,NO,,P;S;X;NI;UN;OT,"P = Principal +S = Secondary +X = Unable to Classify +NI = No information +UN = Unknown +OT = Other","Principal discharge diagnosis flag. Relevant only on IP and IS encounters. + +For ED, AV, and OA encounter types, mark as X = Unable to Classify. (Billing systems do not require a primary diagnosis for ambulatory visits (eg, professional services)) + +One principal diagnosis per encounter is expected, although in some instances more than one diagnosis may be flagged as principal. (Modification made in v3.0 to specify ?per encounter.?)",46 +DIAGNOSIS,RAW_DX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",47 +DIAGNOSIS,RAW_DX_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",48 +DIAGNOSIS,RAW_DX_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",49 +DIAGNOSIS,RAW_PDX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",50 +PROCEDURES,PROCEDURESID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",51 +PROCEDURES,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,52 +PROCEDURES,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. Used to link across tables.,53 +PROCEDURES,ENC_TYPE,RDBMS Text(2),SAS Char(2),,YES,,AV;ED;EI;IP;IS;OA;NI;UN;OT,"AV = Ambulatory Visit +ED = Emergency Department +EI=Emergency Department Admit to Inpatient Hospital Stay (permissible substitution) +IP = Inpatient Hospital Stay +IS = Non-Acute Institutional Stay +OA = Other Ambulatory Visit +NI = No information +UN = Unknown +OT = Other",Please note: This is a field replicated from the ENCOUNTER table. See ENCOUNTER table for definitions.,54 +PROCEDURES,ADMIT_DATE,RDBMS Date,SAS Date (Numeric),,YES,DATE,,,Please note: This is a field replicated from the ENCOUNTER table. See ENCOUNTER table for definitions.,55 +PROCEDURES,PROVIDERID,RDBMS Text(x),SAS Char(x),,YES,,,,Please note: This is a field replicated from the ENCOUNTER table. See ENCOUNTER table for definitions.,56 +PROCEDURES,PX_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"New to v2.0. + +Date the procedure was performed.",57 +PROCEDURES,PX,RDBMS Text(11),SAS Char(11),,NO,,,,Procedure code.,58 +PROCEDURES,PX_TYPE,RDBMS Text(2),SAS Char(2),,NO,,09;10;11;C2;C3;C4;H3;HC;LC;ND;RE;NI;UN;OT,"09 = ICD-9-CM +10 = ICD-10-PCS +11 = ICD-11-PCS +C2 = CPT Category II +C3 = CPT Category III +C4 = CPT-4 (i.e., HCPCS Level I) +H3 = HCPCS Level III +HC = HCPCS (i.e., HCPCS Level II) +LC = LOINC +ND = NDC +RE = Revenue +NI = No information +UN = Unknown +OT = Other","Procedure code type. + +We include a number of code types for flexibility, but the basic requirement that the code refer to a medical procedure remains. + +Revenue codes are a standard concept in Medicare billing and can be useful for defining care settings. If those codes are available they can be included. + +Medications administered by clinicians can be captured in billing data and Electronic Health Records (EHRs) as HCPCS procedure codes. Administration (infusion) of chemotherapy is an example. + +We are now seeing NDCs captured as part of procedures because payers are demanding it for payment authorization. Inclusion of this code type enables those data partners that capture the NDC along with the procedure to include the data. + +Please note: The ?Other? category is meant to identify internal use ontologies and codes.",59 +PROCEDURES,PX_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,OD;BI;CL;NI;UN;OT,"OD=Order +BI=Billing +CL=Claim +NI=No information +UN=Unknown +OT=Other","New to v2.0. + +Source of the procedure information. + +Order and billing pertain to internal healthcare processes and data sources. Claim pertains to data from the bill fulfillment, generally data sources held by insurers and other health plans.",60 +PROCEDURES,RAW_PX,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",61 +PROCEDURES,RAW_PX_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",62 +VITAL,VITALID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique VITAL record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. + +(New field added to v3.0.)",63 +VITAL,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,64 +VITAL,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. This is an optional relationship; the ENCOUNTERID should generally be present if the vitals were measured as part of healthcare delivery captured by this datamart (guidance added in v3.0).,65 +VITAL,MEASURE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date of vitals measure.,66 +VITAL,MEASURE_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Time of vitals measure.,67 +VITAL,VITAL_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,PR;PD;HC;HD;NI;UN;OT,"PR = Patient-reported +PD=Patient device direct feed +HC = Healthcare delivery setting +HD=Healthcare device direct feed +NI = No information +UN = Unknown +OT = Other","Please note: The ?Patient-reported? category can include reporting by patient?s family or guardian. + +v2.0 amendment: The new categorical value of PD and HD have been added. + +v2.0 guidance added with slight modification in v3.0: If unknown whether data are received directly from a device feed, use the more general context (such as patient-reported or healthcare delivery setting).",68 +VITAL,HT,RDBMS Number(8),NUMERIC(8),,NO,INCH,,,"Height (in inches) measured by standing. Only populated if measure was taken on this date. If missing, this value should be null. Decimal precision is permissible. (Modification of wording made from ?blank? to ?null? in v3.0).",69 +VITAL,WT,RDBMS Number(8),NUMERIC(8),,NO,POUND,,,"Weight (in pounds). Only populated if measure was taken on this date. If missing, this value should be null. Decimal precision is permissible. (Modification of wording made from ?blank? to ?null? in v3.0).",70 +VITAL,DIASTOLIC,RDBMS Number(4),NUMERIC(),,NO,MMHG,,,"Diastolic blood pressure (in mmHg). Only populated if measure was taken on this date. If missing, this value should be null. (Modification of wording made from ?blank? to ?null? in v3.0).",71 +VITAL,SYSTOLIC,RDBMS Number(4),NUMERIC(4),,NO,MMHG,,,"Systolic blood pressure (in mmHg). Only populated if measure was taken on this date. If missing, this value should be null. (Modification of wording made from ?blank? to ?null? in v3.0).",72 +VITAL,ORIGINAL_BMI,RDBMS Number(8),NUMERIC(8),,NO,,,,"BMI if calculated in the source system. + +Important: Please do not calculate BMI during CDM implementation. This field should only reflect originating source system calculations, if height and weight are not stored in the source.",73 +VITAL,BP_POSITION,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;NI;UN;OT,"01 = Sitting +02 = Standing +03 = Supine +NI = No information +UN = Unknown +OT = Other",Position for orthostatic blood pressure. This value should be null if blood pressure was not measured. (Modification of wording made from ?blank? to ?null? in v3.0).,74 +VITAL,SMOKING,RDBMS Text(2),SAS Char(2),,,,01;02;03;04;05;06;07;08;NI;UN;OT,"01=Current every day smoker +02=Current some day smoker +03=Former smoker +04=Never smoker +05=Smoker, current status unknown +06=Unknown if ever smoked +07=Heavy tobacco smoker +08=Light tobacco smoker +NI=No information +UN=Unknown +OT=Other","This field is new to v3.0. + +Indicator for any form of tobacco that is smoked. + +Per Meaningful Use guidance, ??smoking status includes any form of tobacco that is smoked, but not all tobacco use.? + +??Light smoker? is interpreted to mean less than 10 cigarettes per day, or an equivalent (but less concretely defined) quantity of cigar or pipe smoke. ?Heavy smoker? is interpreted to mean greater than 10 cigarettes per day or an equivalent (but less concretely defined) quantity of cigar or pipe smoke.? + +??we understand that a ?current every day smoker? or ?current some day smoker? is an individual who has smoked at least 100 cigarettes during his/her lifetime and still regularly smokes every day or periodically, yet consistently; a ?former smoker? would be an individual who has smoked at least 100 cigarettes during his/her lifetime but does not currently smoke; and a ?never smoker? would be an individual who has not smoked 100 or more cigarettes during his/her lifetime.? + +http://www.healthit.gov/sites/default/files/standards-certification/2014-edition-draft-test-procedures/170-314-a-11-smoking-status-2014-test-procedure-draft-v1.0.pdf +[retrieved May 11, 2015] +",75 +VITAL,TOBACCO,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;06;NI;UN;OT,"01=Current user +02=Never +03=Quit/former user +04=Passive or environmental exposure +06=Not asked +NI=No information +UN=Unknown +OT=Other +","This field is new to v2.0 with revised value set and field definition in v3.0. + +Indicator for any form of tobacco.",76 +VITAL,TOBACCO_TYPE,RDBMS Text(2),SAS Char(2),,NO,,"01;02;03;04;05;NI;UN;OT +","01=Smoked tobacco only +02=Non-smoked tobacco only +03=Use of both smoked and non-smoked tobacco products +04=None +05=Use of smoked tobacco but no information about non-smoked tobacco use +NI=No information +UN=Unknown +OT=Other +","This field is new to v2.0, with revised value set in v3.0. + +Type(s) of tobacco used.",77 +VITAL,RAW_DIASTOLIC,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to formatting into the PCORnet CDM.",78 +VITAL,RAW_SYSTOLIC,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to formatting into the PCORnet CDM.",79 +VITAL,RAW_BP_POSITION,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",80 +VITAL,RAW_SMOKING,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set. New to v3.0.",81 +VITAL,RAW_TOBACCO,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set. New to v2.0.",82 +VITAL,RAW_TOBACCO_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set. New to v2.0.",83 +DISPENSING,DISPENSINGID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",84 +DISPENSING,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,85 +DISPENSING,PRESCRIBINGID,RDBMS Text(x),SAS Char(x),,,,,,"This is an optional relationship to the PRESCRIBING table, and may not be generally available. One prescribing order may generate multiple dispensing records. New field added in v3.0.",86 +DISPENSING,DISPENSE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Dispensing date (as close as possible to date the person received the dispensing).,87 +DISPENSING,NDC,RDBMS Text(11),SAS Char(11),,NO,,,,"National Drug Code in the 11-digit, no-dash, HIPAA format. + +Please expunge any place holders (such as dashes or extra digits). + +If needed, guidance on normalization for other forms of NDC can be found: http://www.nlm.nih.gov/research/umls/rxnorm/docs/2012/rxnorm_doco_full_2012-1.html (see section 6)",88 +DISPENSING,DISPENSE_SUP,RDBMS Number(8),Numeric(8),,NO,DAYS,,,"Days supply. Number of days that the medication supports based on the number of doses as reported by the pharmacist. This amount is typically found on the dispensing record. Integer values are expected. + +Important: Please do not calculate during CDM implementation. This field should only reflect originating source system calculations.",89 +DISPENSING,DISPENSE_AMT,RDBMS Number(8),Numeric(8),,NO,UNITS,,,"Number of units (pills, tablets, vials) dispensed. Net amount per NDC per dispensing. This amount is typically found on the dispensing record. Positive values are expected. + +Important: Please do not calculate during CDM implementation. This field should only reflect originating source system calculations. +",90 +DISPENSING,RAW_NDC,RDBMS Text(x),Numeric(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",91 +LAB_RESULT_CM ,LAB_RESULT_CM_ID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique LAB_RESULT_CM record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. + +(New field added to v3.0.)",92 +LAB_RESULT_CM ,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,93 +LAB_RESULT_CM ,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier used to link across tables. This is an optional field, and should only be populated if the lab was collected as part of a healthcare encounter.",94 +LAB_RESULT_CM ,LAB_NAME,RDBMS Text(10),SAS Char(10),,NO,,A1C;CK;CK_MB;CK_MBI;CREATININE;HGB;LDL;INR;TROP_I;TROP_T_QL;TROP_T_QN,"A1C=Hemoglobin A1c +CK=Creatine kinase total +CK_MB=Creatine kinase MB +CK_MBI=Creatine kinase MB/creatine kinase total +CREATININE=Creatinine +HGB=Hemoglobin +LDL=Low-density lipoprotein +INR=International normalized ratio +TROP_I=Troponin I cardiac +TROP_T_QL=Troponin T cardiac (qualitative) +TROP_T_QN=Troponin T cardiac (quantitative)","Laboratory result common measure, a categorical identification for the type of test, which is harmonized across all contributing data partners. + +Please note that it is possible for more than one LOINC code, CPT code, and/or local code to be associated with one LAB_NAME.",95 +LAB_RESULT_CM ,SPECIMEN_SOURCE,RDBMS Text(10),SAS Char(10),,NO,,BLOOD;CSF;PLASMA;PPP;SERUM;SR_PLS;URINE;NI;UN;OT,"BLOOD=blood +CSF=cerebrospinal fluid +PLASMA=plasma +PPP=platelet poor plasma +SERUM=serum +SR_PLS=serum/plasma +URINE=urine +NI=No information +UN=Unknown +OT=Other",Specimen source. All records will have a specimen source; some tests have several possible values for SPECIMEN_SOURCE. Please see the reference tables for additional details.,96 +LAB_RESULT_CM ,LAB_LOINC,RDBMS Text(10),SAS Char(10),,NO,,,,"Logical Observation Identifiers, Names, and Codes (LOINC) from the Regenstrief Institute. Results with local versions of LOINC codes (e.g., LOINC candidate codes) should be included in the RAW_table field, but the LOINC variable should be set to missing. Current LOINC codes are from 3-7 characters long but Regenstrief suggests a length of 10 for future growth. The last digit of the LOINC code is a check digit and is always preceded by a hyphen. All parts of the LOINC code, including the hyphen, must be included. Do not pad the LOINC code with leading zeros. + +Please see the LOINC reference table for known LOINC codes for each LAB_NAME. +",97 +LAB_RESULT_CM ,PRIORITY,RDBMS Text(2),SAS Char(2),,NO,,E;R;S;NI;UN;OT,"E=Expedite +R=Routine +S=Stat +NI=No information +UN=Unknown +OT=Other",Immediacy of test. The intent of this variable is to determine whether the test was obtained as part of routine care or as an emergent/urgent diagnostic test (designated as Stat or Expedite).,98 +LAB_RESULT_CM ,RESULT_LOC,RDBMS Text(2),SAS Char(2),,NO,,L;P;NI;UN;OT,"L=Lab +P=Point of Care +NI=No information +UN=Unknown +OT=Other","Location of the test result. Point of Care locations may include anticoagulation clinic, newborn nursery, finger stick in provider office, or home. The default value is ?L? unless the result is Point of Care. There should not be any missing values.",99 +LAB_RESULT_CM ,LAB_PX,RDBMS Text(11),SAS Char(11),,NO,,,,"Optional variable for local and standard procedure codes, used to identify the originating order for the lab test.",100 +LAB_RESULT_CM ,LAB_PX_TYPE,RDBMS Text(2),SAS Char(2),,NO,,09;10;11;C2;C3;C4;H3;HC;LC;ND;RE;NI;UN;OT,"09=ICD-9-CM +10=ICD-10-PCS +11=ICD-11-PCS +C2=CPT Category II +C3=CPT Category III +C4=CPT-4 (i.e., HCPCS Level I) +H3=HCPCS Level III +HC=HCPCS (i.e., HCPCS Level II) +LC=LOINC +ND=NDC +RE=Revenue +NI=No information +UN=Unknown +OT=Other","Procedure code type, if applicable.",101 +LAB_RESULT_CM ,LAB_ORDER_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date test was ordered.,102 +LAB_RESULT_CM ,SPECIMEN_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Date specimen was collected.,103 +LAB_RESULT_CM ,SPECIMEN_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Time specimen was collected.,104 +LAB_RESULT_CM ,RESULT_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,Result date.,105 +LAB_RESULT_CM ,RESULT_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,Result time.,106 +LAB_RESULT_CM ,RESULT_QUAL,RDBMS Text(12),SAS Char(12),,NO,,"BORDERLINE;POSITIVE;NEGATIVE;UNDETERMINED;NI=No information;UN=Unknown;OT=Other +","BORDERLINE +POSITIVE +NEGATIVE +UNDETERMINED +NI=No information +UN=Unknown +OT=Other",Standardized result for qualitative results. This variable should be NI for quantitative results. Please see the reference tables for additional details and information on acceptable values for each qualitative LAB_NAME.,107 +LAB_RESULT_CM ,RESULT_NUM,RDBMS Number(8),SAS Char(8),,NO,,,,Standardized/converted result for quantitative results. This variable should be null for qualitative results. Please see the reference tables for additional details. (Modification of wording made from ?blank? to ?null? in v3.0).,108 +LAB_RESULT_CM ,RESULT_MODIFIER,RDBMS Text(2),SAS Char(2),,NO,,EQ;GE;GT;LE;LT;TX;NI;UN;OT,"EQ=Equal +GE=Greater than or equal to +GT=Greater than +LE=Less than or equal to +LT=Less than +TX=Text +NI=No information +UN=Unknown +OT=Other","Modifier for result values. Any symbols in the RAW_RESULT value should be reflected in the RESULT_MODIFIER variable. + +For example, if the original source data value is ""<=200"" then RAW_RESULT=200 and RESULT_MODIFIER=LE. If the original source data value is text then RESULT_MODIFIER=TX. If the original source data value is a numeric value then RESULT_MODIFIER=EQ.",109 +LAB_RESULT_CM ,RESULT_UNIT,RDBMS Text(11),SAS Char(11),,NO,,,,Converted/standardized units for the result. Please see the standard abbreviations reference table for additional details.,110 +LAB_RESULT_CM ,NORM_RANGE_LOW,RDBMS Text(10),SAS Char(10),,NO,,,,"Lower bound of the normal range assigned by the laboratory. Value should only contain the value of the lower bound. The symbols >, <, >=, <= should be removed. For example, if the normal range for a test is >100 and <300, then ""100"" should be entered.",111 +LAB_RESULT_CM ,NORM_MODIFIER_LOW,RDBMS Text(2),SAS Char(2),,NO,,EQ;GE;GT;NO;NI;UN;OT,"EQ=Equal +GE=Greater than or equal to +GT=Greater than +NO=No lower limit +NI=No information +UN=Unknown +OT=Other","Modifier for NORM_RANGE_LOW values. + +v3.0 modification made to field name. + +For numeric results one of the following needs to be true: +1) Both MODIFIER_LOW and MODIFIER_HIGH contain EQ (e.g. normal values fall in the range 3-10) +2) MODIFIER_LOW contains GT or GE and MODIFIER_HIGH contains NO (e.g. normal values are >3 with no upper boundary) +3) MODIFIER_HIGH contains LT or LE and MODIFIER_LOW contains NO (e.g. normal values are <=10 with no lower boundary) + +For numeric results one of the following needs to be true: +1) Both MODIFIER_LOW and MODIFIER_HIGH contain EQ (e.g. normal values fall in the range 3-10) +2) MODIFIER_LOW contains GT or GE and MODIFIER_HIGH contains NO (e.g. normal values are >3 with no upper boundary) +3) MODIFIER_HIGH contains LT or LE and MODIFIER_LOW contains NO (e.g. normal values are <=10 with no lower boundary)",112 +LAB_RESULT_CM ,NORM_RANGE_HIGH,RDBMS Text(10),SAS Char(10),,NO,,,,"Upper bound of the normal range assigned by the laboratory. Value should only contain the value of the upper bound. The symbols >, <, >=, <= should be removed. For example, if the normal range for a test is >100 and <300, then ""300"" should be entered.",113 +LAB_RESULT_CM ,NORM_MODIFIER_HIGH,RDBMS Text(2),SAS Char(2),,NO,,EQ;LE;LT;NO;NI;UN;OT,"EQ=Equal +LE=Less than or equal to +LT=Less than +NO=No higher limit +NI=No information +UN=Unknown +OT=Other","Modifier for NORM_RANGE_HIGH values. + +v3.0 modification made to field name. + +For numeric results one of the following needs to be true: +1) Both MODIFIER_LOW and MODIFIER_HIGH contain EQ (e.g. normal values fall in the range 3-10) +2) MODIFIER_LOW contains GT or GE and MODIFIER_HIGH contains NO (e.g. normal values are >3 with no upper boundary) +3) MODIFIER_HIGH contains LT or LE and MODIFIER_LOW contains NO (e.g. normal values are <=10 with no lower boundary)",114 +LAB_RESULT_CM ,ABN_IND,RDBMS Text(2),SAS Char(2),,NO,,AB;AH;AL;CH;CL;CR;IN;NL;NI;UN;OT,"AB=Abnormal +AH=Abnormally high +AL=Abnormally low +CH=Critically high +CL=Critically low +CR=Critical +IN=Inconclusive +NL=Normal +NI=No information +UN=Unknown +OT=Other",Abnormal result indicator. This value comes from the source data; do not apply logic to create it.,115 +LAB_RESULT_CM ,RAW_LAB_NAME,RDBMS Text(x),SAS Char(x),,NO,,,,"Local code related to an individual lab test. This variable will not be used in queries, but may be used by local programmers to associate a record with a particular LAB_NAME.",116 +LAB_RESULT_CM ,RAW_LAB_CODE,RDBMS Text(x),SAS Char(x),,NO,,,,"Local code related to an individual lab test. This variable will not be used in queries, but may be used by local programmers to associate a record with a particular LAB_NAME.",117 +LAB_RESULT_CM ,RAW_PANEL,RDBMS Text(x),SAS Char(x),,NO,,,,"Local code related to a battery or panel of lab tests. This variable will not be used in queries, but may be used by local programmers to associate a record with a particular LAB_NAME.",118 +LAB_RESULT_CM ,RAW_RESULT,RDBMS Text(x),SAS Char(x),,NO,,,,"The original test result value as seen in your source data. Values may include a decimal point, a sign or text (e.g., POSITIVE, NEGATIVE, DETECTED). The symbols >, <, >=, <= should be removed from the value and stored in the Modifier variable instead.",119 +LAB_RESULT_CM ,RAW_UNIT,RDBMS Text(x),SAS Char(x),,NO,,,,Original units for the result in your source data.,120 +LAB_RESULT_CM ,RAW_ORDER_DEPT,RDBMS Text(x),SAS Char(x),,NO,,,,Local code for ordering provider department.,121 +LAB_RESULT_CM ,RAW_FACILITY_CODE,RDBMS Text(x),SAS Char(x),,NO,,,,Local facility code that identifies the hospital or clinic. Taken from facility claims.,122 +CONDITION,CONDITIONID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",123 +CONDITION,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier. Used to link across tables.,124 +CONDITION,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier used to link across tables. This is an optional field, and should only be populated if the item was collected as part of a healthcare encounter. + +If more than one encounter association is present, this field should be populated with the ID of the encounter when the condition was first entered into the system. However, please note that many conditions may be recorded outside of an encounter context. (Guidance added in v3.0.)",125 +CONDITION,REPORT_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date condition was noted, which may be the date when it was recorded by a provider or nurse, or the date on which the patient reported it. Please note that this date may not correspond to onset date. (Additional guidance added in v3.0.)",126 +CONDITION,RESOLVE_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,"Date condition was resolved, if resolution of a transient condition has been achieved. A resolution date is not generally expected for chronic conditions, even if the condition is managed.",127 +CONDITION,ONSET_DATE,RDBMS Date,SAS Date (Numeric),,,,,,"New field added in v3.0. + +Please note that onset date is a very precise concept. Please do not map data unless they precisely match this definition. The REPORT_DATE concept may be a better fit for many systems.",128 +CONDITION,CONDITION_STATUS,RDBMS Text(2),SAS Char(2),,NO,,AC;RS;IN;NI;UN;OT,"AC=Active +RS=Resolved +IN=Inactive +NI=No information +UN=Unknown +OT=Other","Condition status corresponding with REPORT_DATE. + +Guidance: The value of IN=Inactive may be used in situations where a condition is not resolved, but is not currently active (for example, psoriasis).",129 +CONDITION,CONDITION,RDBMS Text(18),SAS Char(18),,NO,,,,"Condition code. + +Leading zeroes and different levels of decimal precision are permissible in this field. Please populate the exact textual value of this diagnosis code, but remove source-specific suffixes and prefixes. Other codes should be listed as recorded in the source data.",130 +CONDITION,CONDITION_TYPE,RDBMS Text(2),SAS Char(2),,NO,,09;10;11;SM;HP;AG;NI;UN;OT,"09=ICD-9-CM +10=ICD-10-CM +11=ICD-11-CM +SM=SNOMED CT +HP=Human Phenotype Ontology +AG=Algorithmic +NI=No information +UN=Unknown +OT=Other","Condition code type. + +Please note: The ?Other? category is meant to identify internal use ontologies and codes. + +v3.0 amendment: The new categorical value of AG has been added. ",131 +CONDITION,CONDITION_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,PR;HC;RG;PC;NI;UN;OT,"PR=Patient-reported medical history +HC=Healthcare problem list +RG=Registry cohort +PC=PCORnet-defined condition algorithm +NI=No information +UN=Unknown +OT=Other","Please note: The ?Patient-reported? category can include reporting by a proxy, such as patient?s family or guardian. + +Guidance: ?Registry cohort? generally refers to cohorts of patients flagged with a certain set of characteristics for management within a health system. + +?Patient-reported? can include self-reported medical history and/or current medical conditions, not captured via healthcare problem lists or registry cohorts. + +v3.0 amendment: The new categorical value of PC has been added.",132 +CONDITION,RAW_CONDITION_STATUS,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",133 +CONDITION,RAW_CONDITION,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",134 +CONDITION,RAW_CONDITION_TYPE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",135 +CONDITION,RAW_CONDITION_SOURCE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",136 +PRO_CM,PRO_CM_ID,RDBMS Text(x),SAS Char(x),,,,,,"Arbitrary identifier for each unique record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID. New field added in v3.0.",137 +PRO_CM,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier for the patient for whom the PRO response was captured. Used to link across tables.,138 +PRO_CM,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary encounter-level identifier used to link across tables. This is an optional field, and should only be populated if the item was collected as part of a healthcare encounter.",139 +PRO_CM,PRO_ITEM,RDBMS Text(7),SAS Char(7),,NO,,PN_0001;PN_0002;PN_0003;PN_0004;PN_0005;PN_0006;PN_0007;PN_0008;PN_0009;PN_0010;PN_0011;PN_0012;PN_0013;PN_0014;PN_0015;PN_0016;PN_0017;PN_0018;PN_0019;PN_0020;PN_0021,"PN_0001=GLOBAL01 +PN_0002=GLOBAL02 +PN_0003=GLOBAL06 +PN_0004=PFA53 +PN_0005=EDDEP29 +PN_0006=HI7 +PN_0007=SLEEP20 +PN_0008=SRPPER11_CAPS +PN_0009=PAININ9 +PN_0010=3793R1 +PN_0011=28676R1 +PN_0012=EOS_P_011 +PN_0013=PEDSGLOBAL2 +PN_0014=PEDSGLOBAL5 +PN_0015=PEDSGLOBAL6 +PN_0016=GLOBAL03 +PN_0017=GLOBAL04 +PN_0018=EDANX53 +PN_0019=SAMHSA +PN_0020=CAHPS 4.0 +PN_0021=PA070",PCORnet identifier for the specific Common Measure item. Please see the Common Measures reference table for more details.,140 +PRO_CM,PRO_LOINC,RDBMS Text(10),SAS Char(10),,NO,,,,"LOINC code for item context and stem. Please see the reference table for known LOINC codes for each common measure. + +Logical Observation Identifiers, Names, and Codes (LOINC) from the Regenstrief Institute. Results with local versions of LOINC codes (e.g., LOINC candidate codes) should be included in the RAW_table field, but the PRO_LOINC variable should be set to missing. Current LOINC codes are from 3-7 characters long but Regenstrief suggests a length of 10 for future growth. The last digit of the LOINC code is a check digit and is always preceded by a hyphen. All parts of the LOINC code, including the hyphen, must be included. Do not pad the LOINC code with leading zeros. +",141 +PRO_CM,PRO_DATE,RDBMS Date,SAS Date (Numeric),,NO,DATE,,,The date of the response.,142 +PRO_CM,PRO_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,TIME,,,The time of the response.,143 +PRO_CM,PRO_RESPONSE,NUMBER(8),Numeric(8),,NO,,,,The numeric response recorded for the item. Please see the Common Measures reference table for the list of valid responses for each item.,144 +PRO_CM,PRO_METHOD,RDBMS Text(2),SAS Char(2),,NO,,PA;EC;PH;IV;NI;UN;OT,"PA=Paper +EC=Electronic +PH=Telephonic +IV=Telephonic with interactive voice response (IVR) technology +NI=No information +UN=Unknown +OT=Other","Method of administration. Electronic includes responses captured via a personal or tablet computer, at web kiosks, or via a smartphone.",145 +PRO_CM,PRO_MODE,RDBMS Text(2),SAS Char(2),,NO,,"SF;SA;PR;PA;NI;UN;OT +","SF=Self without assistance +SA= Self with assistance +PR=Proxy without assistance +PA=Proxy with assistance +NI=No information +UN=Unknown +OT=Other","The person who responded on behalf of the patient for whom the response was captured. A proxy report is a measurement based on a report by someone other than the patient reporting as if he or she is the patient, such as a parent responding for a child, or a caregiver responding for an individual unable to report for themselves. Assistance excludes providing interpretation of the patient?s response. ",146 +PRO_CM,PRO_CAT,RDBMS Text(2),SAS Char(2),,NO,,"Y;N;NI;UN;OT +","Y=Yes +N=No +NI=No information +UN=Unknown +OT=Other",Indicates whether Computer Adaptive Testing (CAT) was used to administer the survey or instrument that the item was part of. May apply to electronic (EC) and telephonic (PH or IV) modes. ,147 +PRO_CM,RAW_PRO_CODE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating code, such as LOINC candidate codes that have not yet been adopted",148 +PRO_CM,RAW_PRO_RESPONSE,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",149 +PRESCRIBING,PRESCRIBINGID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary identifier for each unique PRESCRIBING record. Does not need to be persistent across refreshes, and may be created by methods such as sequence or GUID.",150 +PRESCRIBING,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,151 +PRESCRIBING,ENCOUNTERID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary encounter-level identifier. This is an optional relationship; the ENCOUNTERID should be present if the prescribing activity is directly associated with an encounter.,152 +PRESCRIBING,RX_PROVIDERID,RDBMS Text(x),SAS Char(x),,NO,,,,Provider code for the provider who prescribed the medication. The provider code is a pseudoidentifier with a consistent crosswalk to the real identifier. ,153 +PRESCRIBING,RX_ORDER_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Order date of the prescription by the provider.,154 +PRESCRIBING,RX_ORDER_TIME,RDBMS Text(5),SAS Time (Numeric), HH:MI using 24-hour clock and zero-padding for hour and minute,NO,,,,Order time of the prescription by the provider.,155 +PRESCRIBING,RX_START_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Start date of order. This attribute may not be consistent with the date on which the patient actually begin taking the medication.,156 +PRESCRIBING,RX_END_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,End date of order (if available).,157 +PRESCRIBING,RX_QUANTITY,RDBMS Number(8),Numeric(8),,NO,,,,Quantity ordered.,158 +PRESCRIBING,RX_REFILLS,RDBMS Number(8),Numeric(8),,NO,,,,"Number of refills ordered (not including the original prescription). If no refills are ordered, the value should be zero.",159 +PRESCRIBING,RX_DAYS_SUPPLY,RDBMS Number(8),Numeric(8),,NO,,,,"Number of days supply ordered, as specified by the prescription.",160 +PRESCRIBING,RX_FREQUENCY,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;05;06;07;08;09;NI;UN;OT,"01=Every day +02=Two times a day (BID) +03=Three times a day (TID) +04=Four times a day (QID) +05=Every morning +06=Every afternoon +07=Before meals +08=After meals +09=As needed (PRN) +NI=No information +UN=Unknown +OT=Other +",Specified frequency of medication.,161 +PRESCRIBING,RX_BASIS,RDBMS Text(2),SAS Char(2),,NO,,01;02;NI;UN;OT,"01=Dispensing +02=Administration +NI=No information +UN=Unknown +OT=Other +",Basis of the medication order,162 +PRESCRIBING,RXNORM_CUI,RDBMS Number (8),Numeric(8),,NO,,,,"Where an RxNorm mapping exists for the source medication, this field contains the RxNorm concept identifier (CUI) at the highest possible specificity. + +If more than one option exists for mapping, the following ordered strategy may be adopted: +1)Semantic generic clinical drug +2)Semantic Branded clinical drug +3)Generic drug pack +4)Branded drug pack +",163 +PRESCRIBING,RAW_RX_MED_NAME,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating, full textual medication name from the source.",164 +PRESCRIBING,RAW_RX_FREQUENCY,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",165 +PRESCRIBING,RAW_RXNORM_CUI,RDBMS Text(x),SAS Char(x),,NO,,,,"Optional field for originating value of field, prior to mapping into the PCORnet CDM value set.",166 +PCORNET_TRIAL,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,167 +PCORNET_TRIAL,TRIALID,RDBMS Text(20),SAS Char(20),,NO,,,,Each TRIALID is assigned by the PCORnet trial?s coordinating center.,168 +PCORNET_TRIAL,PARTICIPANTID,RDBMS Text(x),SAS Char(x),,NO,,,,"Arbitrary person-level identifier used to uniquely identify a participant in a PCORnet trial. + +PARTICIPANTID is never repeated or reused for a specific clinical trial, and is generally assigned by trial-specific processes. It may be the same as a randomization ID. +",169 +PCORNET_TRIAL,TRIAL_SITEID,RDBMS Text(x),SAS Char(x),,NO,,,,Each TRIAL_SITEID is assigned by the PCORnet trial coordinating center.,170 +PCORNET_TRIAL,TRIAL_ENROLL_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Date on which the participant enrolled in the trial (generally coincides with trial consent process).,171 +PCORNET_TRIAL,TRIAL_END_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Date on which the participant completes participation in the trial.,172 +PCORNET_TRIAL,TRIAL_WITHDRAW_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,"If applicable, date on which the participant withdraws consent from the trial.",173 +PCORNET_TRIAL,TRIAL_INVITE_CODE,RDBMS Text(20),SAS Char(20),,NO,,,,"Textual strings used to uniquely identify invitations sent to potential participants, and allows acceptances to be associated back to the originating source. + +Where used, there should generally be a unique combination of PATID, TRIAL_NAME, and INVITE_CODE within each datamart. + +For example, this might include ?co-enrollment ID strings? for e-mail invites or ?verification codes? for letter invites. +",174 +DEATH,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,175 +DEATH,DEATH_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Date of death.,176 +DEATH,DEATH_DATE_IMPUTE,RDBMS Text(2),SAS Char(2),,NO,,B;D;M;N;NI;UN;OT,"B=Both month and day imputed +D=Day imputed +M=Month imputed +N=Not imputed +NI=No information +UN=Unknown +OT=Other +","When date of death is imputed, this field indicates which parts of the date were imputed.",177 +DEATH,DEATH_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,L;N;D;S;T;NI;UN;OT,"L=Other, locally defined +N=National Death Index +D=Social Security +S=State Death files +T=Tumor data +NI=No information +UN=Unknown +OT=Other +","Guidance: ?Other, locally defined? may be used to indicate presence of deaths reported from EHR systems, such as in-patient hospital deaths or dead on arrival.",178 +DEATH,DEATH_MATCH_CONFIDENCE,RDBMS Text(2),SAS Char(2),,NO,,E;F;P;NI;UN;OT,"E=Excellent +F=Fair +P=Poor +NI=No information +UN=Unknown +OT=Other +","For situations where a probabilistic patient matching strategy is used, this field indicates the confidence that the patient drawn from external source data represents the actual patient. + +Should not be present where DEATH_SOURCE is L (locally-defined). May not be applicable for DEATH_SOURCE=T (tumor registry data) +",179 +DEATH_CAUSE,PATID,RDBMS Text(x),SAS Char(x),,NO,,,,Arbitrary person-level identifier used to link across tables. ,180 +DEATH_CAUSE,DEATH_CAUSE,RDBMS Text(8),SAS Char(8),,NO,,,,Cause of death code. Please include the decimal point in ICD codes (if any).,181 +DEATH_CAUSE,DEATH_CAUSE_CODE,RDBMS Text(2),SAS Char(2),,NO,,09;10;NI;UN;OT,"09=ICD-9 +10=ICD-10 +NI=No information +UN=Unknown +OT=Other +",Cause of death code type.,182 +DEATH_CAUSE,DEATH_CAUSE_TYPE,RDBMS Text(2),SAS Char(2),,NO,,C;I;O;U;NI;UN;OT,"C=Contributory +I=Immediate/Primary +O=Other +U=Underlying +NI=No information +UN=Unknown +OT=Other +",Cause of death type. There should be only one underlying cause of death.,183 +DEATH_CAUSE,DEATH_CAUSE_SOURCE,RDBMS Text(2),SAS Char(2),,NO,,L;N;D;S;T;NI;UN;OT,"L=Other, locally defined +N=National Death Index +D=Social Security +S=State Death files +T=Tumor data +NI=No information +UN=Unknown +OT=Other +","Source of cause of death information. + +Guidance: ?Other, locally defined? may be used to indicate presence of deaths reported from EHR systems, such as in-patient hospital deaths or dead on arrival. +",184 +DEATH_CAUSE,DEATH_CAUSE_CONFIDENCE,RDBMS Text(2),SAS Char(2),,NO,,E;F;P;NI;UN;OT,"E=Excellent +F=Fair +P=Poor +NI=No information +UN=Unknown +OT=Other +","Confidence in the accuracy of the cause of death based on source, match, number of reporting sources, discrepancies, etc.",185 +HARVEST,NETWORKID,RDBMS Text(10),SAS Char(10),,NO,,,,This identifier is assigned by DSSNI operations,186 +HARVEST,NETWORK_NAME,RDBMS Text(20),SAS Char(20),,NO,,,,Descriptive name of the network,187 +HARVEST,DATAMARTID,RDBMS Text(10),SAS Char(10),,NO,,,,This identifier is assigned by DSSNI operations,188 +HARVEST,DATAMART_NAME,RDBMS Text(20),SAS Char(20),,NO,,,,Descriptive name of the datamart,189 +HARVEST,DATAMART_PLATFORM,RDBMS Text(2),SAS Char(2),,NO,,"01;02;03;04;05;NI;UN;OT +","01=SQL Server +02=Oracle +03=PostgreSQL +04=MySQL +05=SAS +NI=No information +UN=Unknown +OT=Other +",,190 +HARVEST,CDM_VERSION,NUMBER(8),Numeric(8),,NO,,,,"Version currently implemented within this datamart (for example, 1.0, 2.0, 3.0).",191 +HARVEST,DATAMART_CLAIMS,RDBMS Text(2),SAS Char(2),,NO,,01;02;NI;UN;OT,"01=Not present +02=Present +NI=No information +UN=Unknown +OT=Other +",Datamart includes claims data source(s),192 +HARVEST,DATAMART_EHR,RDBMS Text(2),SAS Char(2),,NO,,01;02;NI;UN;OT,"01=Not present +02=Present +NI=No information +UN=Unknown +OT=Other +",Datamart includes EHR data source(s),193 +HARVEST,BIRTH_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the BIRTH_DATE field on the DEMOGRAPHIC table + +Please see notes for additional definitions. +",194 +HARVEST,ENR_START_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the ENR_START_DATE field on the ENROLLMENT table + +Please see notes for additional definitions. +",195 +HARVEST,ENR_END_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the ENR_END_DATE field on the ENROLLMENT table + +Please see notes for additional definitions. +",196 +HARVEST,ADMIT_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the ADMIT_DATE field on the ENCOUNTER table + +Please see notes for additional definitions. +",197 +HARVEST,DISCHARGE_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the DISCHARGE_DATE field on the ENCOUNTER table + +Please see notes for additional definitions. +",198 +HARVEST,PX_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the PX_DATE field on the PROCEDURES table + +Please see notes for additional definitions. +",199 +HARVEST,RX_ORDER_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the RX_ORDER_DATE field on the PRESCRIBING table + +Please see notes for additional definitions. +",200 +HARVEST,RX_START_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the RX_START_DATE field on the PRESCRIBING table + +Please see notes for additional definitions. +",201 +HARVEST,RX_END_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the RX_END_DATE field on the PRESCRIBING table + +Please see notes for additional definitions. +",202 +HARVEST,DISPENSE_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the DISPENSE_DATE field on the DISPENSING table + +Please see notes for additional definitions. +",203 +HARVEST,LAB_ORDER_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the LAB_ORDER_DATE field on the LAB_RESULT_CM table + +Please see notes for additional definitions. +",204 +HARVEST,SPECIMEN_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the SPECIMEN_DATE field on the LAB_RESULT_CM table + +Please see notes for additional definitions. +",205 +HARVEST,RESULT_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the RESULT_DATE field on the LAB_RESULT_CM table + +Please see notes for additional definitions. +",206 +HARVEST,MEASURE_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the MEASURE_DATE field on the VITAL table + +Please see notes for additional definitions. +",207 +HARVEST,ONSET_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the ONSET_DATE field on the CONDITION table + +Please see notes for additional definitions. +",208 +HARVEST,REPORT_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the REPORT_DATE field on the CONDITION table + +Please see notes for additional definitions. +",209 +HARVEST,RESOLVE_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the RESOLVE_DATE field on the CONDITION table + +Please see notes for additional definitions. +",210 +HARVEST,PRO_DATE_MGMT,RDBMS Text(2),SAS Char(2),,NO,,01;02;03;04;NI;UN;OT,"01=No imputation or obfuscation +02=Imputation for incomplete dates +03=Date obfuscation +04=Both imputation and obfuscation NI=No information +UN=Unknown +OT=Other +","Data management strategy currently present in the PRO_DATE field on the PRO_CM table + +Please see notes for additional definitions. +",211 +HARVEST,REFRESH_DEMOGRAPHIC_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DEMOGRAPHIC table. This date should be null if the table does not have records.,212 +HARVEST,REFRESH_ENROLLMENT_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the ENROLLMENT table. This date should be null if the table does not have records.,213 +HARVEST,REFRESH_ENCOUNTER_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the ENCOUNTER table. This date should be null if the table does not have records.,214 +HARVEST,REFRESH_DIAGNOSIS_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DIAGNOSIS table. This date should be null if the table does not have records.,215 +HARVEST,REFRESH_PROCEDURES_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PROCEDURES table. This date should be null if the table does not have records.,216 +HARVEST,REFRESH_VITAL_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the VITAL table. This date should be null if the table does not have records.,217 +HARVEST,REFRESH_DISPENSING_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DISPENSING table. This date should be null if the table does not have records.,218 +HARVEST,REFRESH_LAB_RESULT_CM_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the LAB_RESULT_CM table. This date should be null if the table does not have records.,219 +HARVEST,REFRESH_CONDITION_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the CONDITION table. This date should be null if the table does not have records.,220 +HARVEST,REFRESH_PRO_CM_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PRO_CM table. This date should be null if the table does not have records.,221 +HARVEST,REFRESH_PRESCRIBING_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PRESCRIBING table. This date should be null if the table does not have records.,222 +HARVEST,REFRESH_PCORNET_TRIAL_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the PCORNET_TRIAL table. This date should be null if the table does not have records.,223 +HARVEST,REFRESH_DEATH_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DEATH table. This date should be null if the table does not have records.,224 +HARVEST,REFRESH_DEATH_CAUSE_DATE,RDBMS Date,SAS Date (Numeric),,NO,,,,Most recent date on which the present data were loaded into the DEATH_CAUSE table. This date should be null if the table does not have records.,225 +MED_ADMIN,,,,,,,,,, +PROVIDER,,,,,,,,,, +OBS_CLIN,,,,,,,,,, +OBS_GEN,,,,,,,,,, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..cf04b71 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,132 @@ +## Design: ETL Tasks and SQL Scripts + +The main tasks are in: + + - *epic_flowsheets* -- ETL tasks for Epic Flowsheets to i2b2 + +which is based on a design in: + + - *etl_tasks* -- Source-agnostic Luigi ETL Task support + - *script_lib* -- library of SQL scripts + - *sql_syntax* -- break SQL scripts into statements, etc. + +Tasks such as `epic_flowsheets.FlowsheetViews` are based on SQL +scripts such as `sql_scripts/epic_flowsheets_transform.sql` wrapped in +a `etl_tasks.SqlScriptTask`. + + +### SQL Script Library Design, Style and Conventions + +Each script should start with a header comment and some +dependency-checking queries. See `script_lib.py` for details. + +SQL should be written in lowercase, indented 2 spaces, 120 maximum line +length. More details are in the evolving `sql-style.xml` sqldeveloper +style profile. + + - *ISSUE*: sqldeveloper 3 vs. 4 style files? + +See also notes on value enumerations in the header of `sql_scripts/cms_keys.pls`. +*TODO*: port those notes from GROUSE. + + +## Python doctest for story telling and unit testing + +Each python module header should tell a story using [doctest][], +i.e. examples that are also unit tests. + +You can run them one module at a time: + + (luigi)$ python -m doctest script_lib.py -v + Trying: + Script.migrate_fact_upload.title + Expecting: + 'append data from a workspace table.' + ... + 22 tests in 13 items. + 22 passed and 0 failed. + Test passed. + +Or install [nose][] and run all modules at once: + + (grouse-etl)% nosetests --with-doctest + ...................... + ---------------------------------------------------------------------- + Ran 22 tests in 0.658s + + OK + +[doctest]: http://docs.python.org/2/library/doctest.html +[nose]: https://pypi.python.org/pypi/nose/ + + +## Python code style + +We appreciate object capability discipline and the "don't call us, +we'll call you" style that facilitates unit testing with mocks. + + - *ISSUE*: Luigi's design doesn't seem to support this idiom. + Constructors are implict and tasks parameters have to be + serializable, which works against the usual closure + object pattern. Also, the task cache is global mutable + state. + +We avoid mutable state, preferring functional style. + + - *NOTE*: PEP8 tools warn against assinging a lambda to a name, + suggesting `def` instead. We're fine with it; hence + `ignore = E731` in `setup.cfg`. + + +We follow PEP8. The first line of a module or function docstring +should be a short description; if it is blank, either the item is in +an early stage of development or the name and doctests are supposed to +make the purpose obvious. + + - *NOTE*: with static type annotations, the 79 character line + length limit is awkward; hence we use 99 in `setup.cfg`. + + - *ISSUE*: Dan didn't realize until recently that PEP8 recommends + triple double quotes over triple single quotes for + docstrings. He's in the habit of using single quotes + to minimize use of the shift key. + + +## Checking the code + +Once dependencies in `requirements.txt` are satisfied, code should +pass tests, style checks, and static type checking: + + $ nosetests && flake8 . && mypy . + +_tested with mypy-0.720_ + +### Checking in emacs + +To check with `M-x compile` in emacs, first use `M-x pyvenv-activate` +from the [pyvenv][] package. + +To check continuously as you edit, use [flycheck][] and activate +likewise. + +[pyvenv]: https://melpa.org/#/pyvenv +[pyvent]: https://melpa.org/#/flycheck + + +## Luigi Troubleshooting + +**ISSUE**: why won't luigi find modules in the current directory? + Use `PYTHONPATH=. luigi ...` if necessary. + +Most diagnostics are self-explanatory; `etl_tasks` includes +`SQLScriptError` and `ConnectionProblem` exception classes intended to +improve diagnostics + +One challenging diagnostic is: + + RuntimeError: Unfulfilled dependency at run time: DiagnosesLoad_oracle___dconnol_CMS_DEID_SAMPLE_1438246788671_bd6231c982 + +It seems to indicate that the `.complete()` test on a required task +fails even after that task has been `.run()`. For example, the `select +count(*)` completion test in a load script might have failed because +of incorrect join constraints. diff --git a/LICENSE b/LICENSE index 61a4be2..c4cf5f0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,25 @@ +Copyright (c) 2014-2018 Univeristy of Kansas Medical Center + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + i2b2 Software License (“Software License”) Version 2.1 diff --git a/Oracle/PCORI_MEDS_SCHEMA_CHANGE_ora.sql b/Oracle/PCORI_MEDS_SCHEMA_CHANGE_ora.sql deleted file mode 100644 index 139f479..0000000 --- a/Oracle/PCORI_MEDS_SCHEMA_CHANGE_ora.sql +++ /dev/null @@ -1,50 +0,0 @@ -alter table pcornet_med add PCORI_NDC varchar2(12); -UPDATE PCORNET_MED set pcori_ndc = pcori_basecode where length(pcori_basecode)=11 and c_hlevel>2 and LOWER(sourcesystem_cd) not in ('integration_tool') -and pcori_basecode not like 'N%'; - -alter table pcornet_med add PCORI_CUI varchar2(8); -UPDATE pcornet_med set PCORI_CUI = PCORI_BASECODE where length(pcori_basecode)<11 and c_hlevel>2 and LOWER(sourcesystem_cd) not in ('integration_tool') -and pcori_basecode not like 'N%' and m_applied_path='@'; - - -create table CUI_T as -select * from -( -with -cui(c_fullname,pcori_cui,c_hlevel) as -( - select c_fullname,pcori_cui,c_hlevel from pcornet_med where pcori_cui is not null - union all - select m.c_fullname, cui.pcori_cui, m.c_hlevel - from pcornet_med m inner join cui on cui.c_fullname=m.c_path where m.pcori_cui is null -), -cuid as -( select c_fullname, pcori_cui, row_number() over (partition by C_FULLNAME order by c_hlevel desc)as rowno from cui) - -select DISTINCT cuid.c_fullname as c_fullname, cuid.pcori_cui as pcori_cui from cuid where cuid.rowno=1 -); - -update pcornet_med x set x.pcori_cui = - (select pcori_cui from CUI_T t where x.c_fullname = t.c_fullname) -where x.pcori_cui is null; - - - -create table NDC_T as -select * from -( -with ndc(c_fullname,pcori_ndc,c_hlevel) as -( - select c_fullname,pcori_ndc,c_hlevel from pcornet_med where pcori_ndc is not null - union all - select m.c_fullname,ndc.pcori_ndc,m.c_hlevel from pcornet_med m - inner join ndc on ndc.c_fullname=m.c_path where m.pcori_ndc is null -), -ndcd as ( select c_fullname,pcori_ndc, row_number() over (partition by C_FULLNAME order by c_hlevel desc) row from ndc) - -select DISTINCT ndcd.c_fullname as c_fullname, ndcd.pcori_ndc as pcori_ndc from ndcd where ndcd.row=1 -); - -update pcornet_med x set x.pcori_ndc = - (select pcori_ndc from NDC_T t where x.c_fullname = t.c_fullname) -where x.pcori_ndc is null; \ No newline at end of file diff --git a/Oracle/PCORNetDrop.sql b/Oracle/PCORNetDrop.sql new file mode 100644 index 0000000..39c1a1e --- /dev/null +++ b/Oracle/PCORNetDrop.sql @@ -0,0 +1,40 @@ +BEGIN +PMN_DROPSQL('DROP TABLE demographic'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE enrollment'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE encounter'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE drg'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE diagnosis'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE sourcefact'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE pdxfact'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE originfact'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE procedures'); +END; +/ +BEGIN +PMN_DROPSQL('DROP TABLE vital'); +END; +/ \ No newline at end of file diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql new file mode 100644 index 0000000..2cfa68c --- /dev/null +++ b/Oracle/PCORNetInit.sql @@ -0,0 +1,1009 @@ +------------------------------------------------------------------------------------------- +-- PCORNetInit Script +-- +-- This script prepares target load and intermidiary transform tables, helper +-- functions and procedure, and synonyms and views of the source i2b2 source +-- tables. +-- +-- This script should be run as the initial step of running the i2p-transform, +-- but has been seperated out from PCORNetLoader_ora.sql so that the former can +-- be run without dropping all tables. +-- +-- Created by: Michael Prittie (mprittie@kumc.edu) +-- Adapted from original PCORNetLoader_ora.sql +-------------------------------------------------------------------------------- + + +-------------------------------------------------------------------------------- +-- HELPER FUNCTIONS AND PROCEDURES +-------------------------------------------------------------------------------- + +create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS + BEGIN + DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. + tabname => table_name, + estimate_percent => 50, -- Percentage picked somewhat arbitrarily + cascade => TRUE, + degree => 16 + ); +END GATHER_TABLE_STATS; +/ + + +create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS + BEGIN + EXECUTE IMMEDIATE sqlstring; + EXCEPTION + WHEN OTHERS THEN NULL; +END PMN_DROPSQL; +/ + + +create or replace FUNCTION PMN_IFEXISTS(objnamestr VARCHAR2, objtypestr VARCHAR2) RETURN BOOLEAN AS +cnt NUMBER; +BEGIN + SELECT COUNT(*) + INTO cnt + FROM USER_OBJECTS + WHERE upper(OBJECT_NAME) = upper(objnamestr) + and upper(object_type) = upper(objtypestr); + + IF( cnt = 0 ) + THEN + --dbms_output.put_line('NO!'); + return FALSE; + ELSE + --dbms_output.put_line('YES!'); + return TRUE; + END IF; + +END PMN_IFEXISTS; +/ + + +create or replace PROCEDURE PMN_Execuatesql(sqlstring VARCHAR2) AS +BEGIN + EXECUTE IMMEDIATE sqlstring; + dbms_output.put_line(sqlstring); +END PMN_ExecuateSQL; +/ + + +--ACK: http://dba.stackexchange.com/questions/9441/how-to-catch-and-handle-only-specific-oracle-exceptions +create or replace procedure create_error_table(table_name varchar2) as +sqltext varchar2(4000); + +begin + dbms_errlog.create_error_log(dml_table_name => table_name); +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -955 THEN + NULL; -- suppresses ORA-00955 exception ("name is already used by an existing object") + ELSE + RAISE; + END IF; +-- Delete rows from a previous run in case the table already existed +sqltext := 'delete from ERR$_' || table_name; +PMN_Execuatesql(sqltext); +end; +/ + + +create or replace FUNCTION GETDATAMARTID RETURN VARCHAR2 IS +BEGIN + RETURN '&&datamart_id'; +END; +/ + + +CREATE OR REPLACE FUNCTION GETDATAMARTNAME RETURN VARCHAR2 AS +BEGIN + RETURN '&&datamart_name'; +END; +/ + + +CREATE OR REPLACE FUNCTION GETDATAMARTPLATFORM RETURN VARCHAR2 AS +BEGIN + RETURN '02'; -- 01 is MSSQL, 02 is Oracle +END; +/ + + +BEGIN +PMN_DROPSQL('DROP TABLE pcornet_codelist'); +END; +/ +create table pcornet_codelist(codetype varchar2(20), code varchar2(50)) +/ + +create or replace procedure pcornet_parsecode (codetype in varchar, codestring in varchar) as + +tex varchar(2000); +pos number(9); +readstate char(1) ; +nextchar char(1) ; +val varchar(50); + +begin + +val:=''; +readstate:='F'; +pos:=0; +tex := codestring; +FOR pos IN 1..length(tex) +LOOP +-- dbms_output.put_line(val); + nextchar:=substr(tex,pos,1); + if nextchar!=',' then + if nextchar='''' then + if readstate='F' then + val:=''; + readstate:='T'; + else + insert into pcornet_codelist values (codetype,val); + val:=''; + readstate:='F' ; + end if; + else + if readstate='T' then + val:= val || nextchar; + end if; + end if; + end if; +END LOOP; + +end pcornet_parsecode; +/ + +create or replace procedure pcornet_popcodelist as + +codedata varchar(2000); +onecode varchar(20); +codetype varchar(20); + +cursor getcodesql is +select 'RACE',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' +union +select 'SEX',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' +union +select 'HISPANIC',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%'; + +begin +open getcodesql; +LOOP + fetch getcodesql into codetype,codedata; + EXIT WHEN getcodesql%NOTFOUND ; + pcornet_parsecode (codetype,codedata ); +end loop; + +close getcodesql ; +end pcornet_popcodelist; +/ + + +-------------------------------------------------------------------------------- +-- I2B2 SYNONYMS, VIEWS, AND INTERMEDIARY TABLES +-------------------------------------------------------------------------------- + + +CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT +/ + +CREATE OR REPLACE SYNONYM I2B2MEDFACT FOR OBSERVATION_FACT_MEDS +/ + +BEGIN +PMN_DROPSQL('DROP TABLE i2b2patient_list'); +END; +/ + +CREATE table i2b2patient_list as +select * from +( +select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') +) where ROWNUM<100000000 +/ + +create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) +/ + +create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE to_date('01-Jan-2010','dd-mon-rrrr') -) where ROWNUM<100000000 -/ - -create or replace VIEW i2b2patient as select * from I2B2DEMODATA.PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) -/ - -create or replace view i2b2visit as select * from I2B2DEMODATA.VISIT_DIMENSION where START_DATE >= to_date('01-Jan-2010','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE = (select cutoff from icd10_transition) then 0 + when diag.dx_type = (select icd_10_cm from dx_type) + and factline.start_date < (select cutoff from icd10_transition) then 0 + else 1 + end) asc) unique_row + from i2b2fact factline + join diag on diag.c_basecode = factline.concept_cd +) +, diag_fact_cutoff_filter as ( + select * from diag_fact_merge where unique_row = 1 +) +select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, enc.admit_date, enc.providerid + , factline.pcori_basecode dx + , factline.dx_type dxtype, + CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END dx_source, + nvl(SUBSTR(originsource,INSTR(originsource, ':')+1,2),'NI') dx_origin, + CASE WHEN enc_type in ('EI', 'IP', 'IS') -- PDX is "relevant only on IP and IS encounters" + THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, ':')+1,2),'NI') + ELSE 'X' END PDX +from diag_fact_cutoff_filter factline +inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + left outer join sourcefact +on factline.patient_num=sourcefact.patient_num +and factline.encounter_num=sourcefact.encounter_num +and factline.provider_id=sourcefact.provider_id +and factline.concept_cd=sourcefact.concept_Cd +and factline.start_date=sourcefact.start_Date +left outer join pdxfact +on factline.patient_num=pdxfact.patient_num +and factline.encounter_num=pdxfact.encounter_num +and factline.provider_id=pdxfact.provider_id +and factline.concept_cd=pdxfact.concept_cd +and factline.start_date=pdxfact.start_Date +left outer join originfact --CDM 3.1 addition +on factline.patient_num=originfact.patient_num +and factline.encounter_num=originfact.encounter_num +and factline.provider_id=originfact.provider_id +and factline.concept_cd=originfact.concept_cd +and factline.start_date=originfact.start_Date +where (sourcefact.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\%' or sourcefact.c_fullname is null) +-- order by enc.admit_date desc +; + +execute immediate 'create index diagnosis_idx on diagnosis (PATID, ENCOUNTERID)'; +GATHER_TABLE_STATS('DIAGNOSIS'); end PCORNetDiagnosis; / - - create or replace procedure PCORNetCondition as -sqltext varchar2(4000); begin - -PMN_DROPSQL('DROP TABLE sourcefact2'); - -sqltext := 'create table sourcefact2 as '|| - 'select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname '|| - 'from i2b2fact factline '|| - 'inner join pmnENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| - 'inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode '|| - 'where dxsource.c_fullname like ''\PCORI_MOD\CONDITION_OR_DX\%'''; -PMN_EXECUATESQL(sqltext); - -sqltext := 'insert into pmncondition (patid, encounterid, report_date, resolve_date, condition, condition_type, condition_status, condition_source) '|| -'select distinct factline.patient_num, min(factline.encounter_num) encounterid, min(factline.start_date) report_date, NVL(max(factline.end_date),null) resolve_date, diag.pcori_basecode, '|| -'SUBSTR(diag.c_fullname,18,2) condition_type, '|| -' NVL2(max(factline.end_date) , ''RS'', ''NI'') condition_status, '|| -- Imputed so might not be entirely accurate -' NVL(SUBSTR(max(dxsource),INSTR(max(dxsource), '':'')+1,2),''NI'') condition_source '|| -'from i2b2fact factline '|| -'inner join pmnENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num '|| -'inner join pcornet_diag diag on diag.c_basecode = factline.concept_cd '|| -' left outer join sourcefact2 sf '|| -'on factline.patient_num=sf.patient_num '|| -'and factline.encounter_num=sf.encounter_num '|| -'and factline.provider_id=sf.provider_id '|| -'and factline.concept_cd=sf.concept_Cd '|| -'and factline.start_date=sf.start_Date '|| -'where diag.c_fullname like ''\PCORI\DIAGNOSIS\%'' '|| -'and sf.c_fullname like ''\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\%'' '|| -'group by factline.patient_num, diag.pcori_basecode, diag.c_fullname '; - -PMN_EXECUATESQL(sqltext); +PMN_DROPSQL('drop index condition_idx'); +PMN_DROPSQL('drop index sourcefact2_idx'); + +execute immediate 'truncate table condition'; +execute immediate 'truncate table sourcefact2'; + +insert into sourcefact2 + select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname + from i2b2fact factline + inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + where dxsource.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\%'; + +execute immediate 'create index sourcefact2_idx on sourcefact2 (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('SOURCEFACT2'); + +create_error_table('CONDITION'); + +insert into condition (patid, encounterid, report_date, resolve_date, condition, condition_type, condition_status, condition_source) +select distinct factline.patient_num, min(factline.encounter_num) encounterid, min(factline.start_date) report_date, NVL(max(factline.end_date),null) resolve_date, diag.pcori_basecode, +SUBSTR(diag.c_fullname,18,2) condition_type, + NVL2(max(factline.end_date) , 'RS', 'NI') condition_status, -- Imputed so might not be entirely accurate + NVL(SUBSTR(max(dxsource),INSTR(max(dxsource), ':')+1,2),'NI') condition_source +from i2b2fact factline +inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num +inner join pcornet_diag diag on diag.c_basecode = factline.concept_cd + left outer join sourcefact2 sf +on factline.patient_num=sf.patient_num +and factline.encounter_num=sf.encounter_num +and factline.provider_id=sf.provider_id +and factline.concept_cd=sf.concept_Cd +and factline.start_date=sf.start_Date +where diag.c_fullname like '\PCORI\DIAGNOSIS\%' +and sf.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\%' +group by factline.patient_num, diag.pcori_basecode, diag.c_fullname +log errors into ERR$_CONDITION reject limit unlimited +; + +execute immediate 'create index condition_idx on condition (PATID, ENCOUNTERID)'; +GATHER_TABLE_STATS('CONDITION'); end PCORNetCondition; / +create or replace procedure PCORNetProcedure as +begin +PMN_DROPSQL('drop index procedures_idx'); +execute immediate 'truncate table procedures'; - -create or replace procedure PCORNetProcedure as -begin -insert into pmnprocedure( - patid, encounterid, enc_type, admit_date, providerid, px, px_type) -select distinct fact.patient_num, enc.encounterid, enc.enc_type, fact.start_date, - fact.provider_id, SUBSTR(pr.pcori_basecode,INSTR(pr.pcori_basecode, ':')+1,11) px, SUBSTR(pr.c_fullname,18,2) pxtype +insert into procedures( + patid, encounterid, enc_type, admit_date, px_date, providerid, px, px_type, px_source) +select distinct fact.patient_num, enc.encounterid, enc.enc_type, enc.admit_date, fact.start_date, + fact.provider_id, SUBSTR(pr.pcori_basecode,INSTR(pr.pcori_basecode, ':')+1,11) px, SUBSTR(pr.c_fullname,18,2) pxtype, + -- All are billing for now - see https://informatics.gpcnetwork.org/trac/Project/ticket/491 + 'BI' px_source from i2b2fact fact - inner join pmnENCOUNTER enc on enc.patid = fact.patient_num and enc.encounterid = fact.encounter_Num - inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd + inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd + inner join encounter enc on enc.patid = fact.patient_num and enc.encounterid = fact.encounter_Num where pr.c_fullname like '\PCORI\PROCEDURE\%'; +execute immediate 'create index procedures_idx on procedures (PATID, ENCOUNTERID)'; +GATHER_TABLE_STATS('PROCEDURES'); + end PCORNetProcedure; / @@ -1091,8 +500,13 @@ end PCORNetProcedure; create or replace procedure PCORNetVital as begin + +PMN_DROPSQL('drop index vital_idx'); + +execute immediate 'truncate table vital'; + -- jgk: I took out admit_date - it doesn't appear in the scheme. Now in SQLServer format - date, substring, name on inner select, no nested with. Added modifiers and now use only pathnames, not codes. -insert into pmnVITAL(patid, encounterid, measure_date, measure_time,vital_source,ht, wt, diastolic, systolic, original_bmi, bp_position,smoking,tobacco,tobacco_type) +insert into vital(patid, encounterid, measure_date, measure_time,vital_source,ht, wt, diastolic, systolic, original_bmi, bp_position,smoking,tobacco,tobacco_type) select patid, encounterid, to_date(measure_date,'rrrr-mm-dd') measure_date, measure_time,vital_source,ht, wt, diastolic, systolic, original_bmi, bp_position,smoking,tobacco, case when tobacco in ('02','03','04') then -- no tobacco case when smoking in ('03','04') then '04' -- no smoking @@ -1122,12 +536,12 @@ from ( , case when vit.pcori_code like '\PCORI\VITAL\TOBACCO\SMOKING\%' then vit.pcori_basecode else null end smoking , case when vit.pcori_code like '\PCORI\VITAL\TOBACCO\__\%' then vit.pcori_basecode else null end unk_tobacco , enc.admit_date - from pmndemographic pd + from demographic pd left join ( select obs.patient_num patid, obs.encounter_num encounterid, to_char(obs.start_Date,'YYYY-MM-DD') measure_date, - to_char(obs.start_Date,'HH:MI') measure_time, + to_char(obs.start_Date,'HH24:MI') measure_time, nval_num, pcori_basecode, codes.pcori_code from i2b2fact obs inner join (select c_basecode concept_cd, c_fullname pcori_code, pcori_basecode @@ -1151,7 +565,7 @@ from ( where pm.c_fullname like bp.concept_path || '%' ) codes on codes.concept_cd = obs.concept_cd ) vit on vit.patid = pd.patid - join pmnencounter enc on enc.encounterid = vit.encounterid + join encounter enc on enc.encounterid = vit.encounterid ) x where ht is not null or wt is not null @@ -1164,6 +578,9 @@ where ht is not null or tobacco is not null group by patid, encounterid, measure_date, measure_time, admit_date) y; +execute immediate 'create index vital_idx on vital (PATID)'; +GATHER_TABLE_STATS('VITAL'); + end PCORNetVital; / @@ -1175,44 +592,69 @@ end PCORNetVital; create or replace procedure PCORNetEnroll as begin -INSERT INTO pmnENROLLMENT(PATID, ENR_START_DATE, ENR_END_DATE, CHART, BASIS) - select x.patient_num patid, case when l.patient_num is not null then l.period_start else enr_start end enr_start_date - , case when l.patient_num is not null then l.period_end when enr_end_end>enr_end then enr_end_end else enr_end end enr_end_date - , 'Y' chart, case when l.patient_num is not null then 'A' else 'E' end basis from - (select patient_num, min(start_date) enr_start,max(start_date) enr_end,max(end_date) enr_end_end from i2b2visit where patient_num in (select patid from pmndemographic) group by patient_num) x - left outer join i2b2loyalty_patients l on l.patient_num=x.patient_num; +PMN_DROPSQL('drop index enrollment_idx'); + +execute immediate 'truncate table enrollment'; + +INSERT INTO enrollment(PATID, ENR_START_DATE, ENR_END_DATE, CHART, ENR_BASIS) +with pats_delta as ( + -- If only one visit, visit_delta_days will be 0 + select patient_num, max(start_date) - min(start_date) visit_delta_days + from i2b2visit + where start_date > add_months(sysdate, -&&enrollment_months_back) + group by patient_num + ), +enrolled as ( + select distinct patient_num + from pats_delta + where visit_delta_days > 30 + ) +select + visit.patient_num patid, min(visit.start_date) enr_start_date, + max(visit.start_date) enr_end_date, 'Y' chart, 'A' enr_basis +from enrolled enr +join i2b2visit visit on enr.patient_num = visit.patient_num +group by visit.patient_num; + +execute immediate 'create index enrollment_idx on enrollment (PATID)'; +GATHER_TABLE_STATS('ENROLLMENT'); end PCORNetEnroll; / create or replace procedure PCORNetLabResultCM as -sqltext varchar2(4000); begin -PMN_DROPSQL('DROP TABLE priority'); -sqltext := 'create table priority as '|| -'(select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode PRIORITY '|| -'from i2b2fact '|| -'inner join pmnENCOUNTER enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num '|| -'inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode '|| -'where c_fullname LIKE ''\PCORI_MOD\PRIORITY\%'') '; +PMN_DROPSQL('drop index lab_result_cm_idx'); +PMN_DROPSQL('drop index priority_idx'); +PMN_DROPSQL('drop index location_idx'); -PMN_EXECUATESQL(sqltext); +execute immediate 'truncate table priority'; +execute immediate 'truncate table location'; +execute immediate 'truncate table lab_result_cm'; +insert into priority +select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode PRIORITY +from i2b2fact +inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num +inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode +where c_fullname LIKE '\PCORI_MOD\PRIORITY\%'; -PMN_DROPSQL('DROP TABLE location'); -sqltext := 'create table location as '|| -'(select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode RESULT_LOC '|| -'from i2b2fact '|| -'inner join pmnENCOUNTER enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num '|| -'inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode '|| -'where c_fullname LIKE ''\PCORI_MOD\RESULT_LOC\%'') '; +execute immediate 'create index priority_idx on priority (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('PRIORITY'); -PMN_EXECUATESQL(sqltext); +insert into location +select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, lsource.pcori_basecode RESULT_LOC +from i2b2fact +inner join encounter enc on enc.patid = i2b2fact.patient_num and enc.encounterid = i2b2fact.encounter_Num +inner join pcornet_lab lsource on i2b2fact.modifier_cd =lsource.c_basecode +where c_fullname LIKE '\PCORI_MOD\RESULT_LOC\%'; +execute immediate 'create index location_idx on location (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('LOCATION'); -INSERT INTO pmnlabresults_cm +INSERT INTO lab_result_cm (PATID ,ENCOUNTERID ,LAB_NAME @@ -1244,10 +686,9 @@ INSERT INTO pmnlabresults_cm ,RAW_ORDER_DEPT ,RAW_FACILITY_CODE) - SELECT DISTINCT M.patient_num patid, M.encounter_num encounterid, -CASE WHEN ont_parent.C_BASECODE LIKE 'LAB_NAME%' then SUBSTR (ont_parent.c_basecode,10, 10) ELSE 'NI' END LAB_NAME, +CASE WHEN ont_parent.C_BASECODE LIKE 'LAB_NAME%' then SUBSTR (ont_parent.c_basecode,10, 10) ELSE 'UN' END LAB_NAME, CASE WHEN lab.pcori_specimen_source like '%or SR_PLS' THEN 'SR_PLS' WHEN lab.pcori_specimen_source is null then 'NI' ELSE lab.pcori_specimen_source END specimen_source, -- (Better way would be to fix the column in the ontology but this will work) NVL(lab.pcori_basecode, 'NI') LAB_LOINC, NVL(p.PRIORITY,'NI') PRIORITY, @@ -1256,32 +697,49 @@ NVL(lab.pcori_basecode, 'NI') LAB_PX, 'LC' LAB_PX_TYPE, m.start_date LAB_ORDER_DATE, m.start_date SPECIMEN_DATE, -to_char(m.start_date,'HH:MI') SPECIMEN_TIME, +to_char(m.start_date,'HH24:MI') SPECIMEN_TIME, m.end_date RESULT_DATE, -to_char(m.end_date,'HH:MI') RESULT_TIME, -CASE WHEN m.ValType_Cd='T' THEN NVL(nullif(m.TVal_Char,''),'NI') ELSE 'NI' END RESULT_QUAL, -- TODO: Should be a standardized value +to_char(m.end_date,'HH24:MI') RESULT_TIME, +--CASE WHEN m.ValType_Cd='T' THEN NVL(nullif(m.TVal_Char,''),'NI') ELSE 'NI' END RESULT_QUAL, -- TODO: Should be a standardized value +'NI' RESULT_QUAL, -- Local fix for KUMC (temp) CASE WHEN m.ValType_Cd='N' THEN m.NVAL_NUM ELSE null END RESULT_NUM, CASE WHEN m.ValType_Cd='N' THEN (CASE NVL(nullif(m.TVal_Char,''),'NI') WHEN 'E' THEN 'EQ' WHEN 'NE' THEN 'OT' WHEN 'L' THEN 'LT' WHEN 'LE' THEN 'LE' WHEN 'G' THEN 'GT' WHEN 'GE' THEN 'GE' ELSE 'NI' END) ELSE 'TX' END RESULT_MODIFIER, -NVL(m.Units_CD,'NI') RESULT_UNIT, -- TODO: Should be standardized units -nullif(norm.NORM_RANGE_LOW,'') NORM_RANGE_LOW -,norm.NORM_MODIFIER_LOW, -nullif(norm.NORM_RANGE_HIGH,'') NORM_RANGE_HIGH -,norm.NORM_MODIFIER_HIGH, +--NVL(m.Units_CD,'NI') RESULT_UNIT, -- TODO: Should be standardized units +CASE + WHEN INSTR(m.Units_CD, '%') > 0 THEN 'PERCENT' + WHEN m.Units_CD IS NULL THEN NVL(m.Units_CD,'NI') + when length(m.Units_CD) > 11 then substr(m.Units_CD, 1, 11) + ELSE TRIM(REPLACE(UPPER(m.Units_CD), '(CALC)', '')) +end RESULT_UNIT, -- Local fix for KUMC +norm.ref_lo NORM_RANGE_LOW, +case + when norm.ref_lo is not null and norm.ref_hi is not null then 'EQ' + when norm.ref_lo is not null and norm.ref_hi is null then 'GE' + when norm.ref_lo is null and norm.ref_hi is not null then 'NO' +else 'NI' +end NORM_MODIFIER_LOW, +norm.ref_hi NORM_RANGE_HIGH, +case + when norm.ref_lo is not null and norm.ref_hi is not null then 'EQ' + when norm.ref_lo is not null and norm.ref_hi is null then 'NO' + when norm.ref_lo is null and norm.ref_hi is not null then 'LE' + else 'NI' +end NORM_MODIFIER_HIGH, CASE NVL(nullif(m.VALUEFLAG_CD,''),'NI') WHEN 'H' THEN 'AH' WHEN 'L' THEN 'AL' WHEN 'A' THEN 'AB' ELSE 'NI' END ABN_IND, NULL RAW_LAB_NAME, NULL RAW_LAB_CODE, NULL RAW_PANEL, -CASE WHEN m.ValType_Cd='T' THEN m.TVal_Char ELSE to_char(m.NVal_Num) END RAW_RESULT, +--CASE WHEN m.ValType_Cd='T' THEN m.TVal_Char ELSE to_char(m.NVal_Num) END RAW_RESULT, +CASE WHEN m.ValType_Cd='T' THEN substr(m.TVal_Char, 1, 50) ELSE to_char(m.NVal_Num) END RAW_RESULT, -- Local fix for KUMC NULL RAW_UNIT, NULL RAW_ORDER_DEPT, -NULL RAW_FACILITY_CODE +m.concept_cd RAW_FACILITY_CODE FROM i2b2fact M -inner join pmnENCOUNTER enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num -- Constraint to selected encounters - +inner join encounter enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num -- Constraint to selected encounters +inner join demographic demo on demo.patid=m.patient_num inner join pcornet_lab lab on lab.c_basecode = M.concept_cd and lab.c_fullname like '\PCORI\LAB_RESULT_CM\%' inner JOIN pcornet_lab ont_parent on lab.c_path=ont_parent.c_fullname -inner join pmn_labnormal norm on ont_parent.c_basecode=norm.LAB_NAME LEFT OUTER JOIN priority p @@ -1298,10 +756,20 @@ and M.encounter_num=l.encounter_num and M.provider_id=l.provider_id and M.concept_cd=l.concept_Cd and M.start_date=l.start_Date + +LEFT OUTER JOIN labnormal norm + on m.concept_cd=norm.concept_cd + and demo.sex=norm.sex + and (m.start_date - demo.birth_date) > norm.age_lower + and (m.start_date - demo.birth_date) <= norm.age_upper WHERE m.ValType_Cd in ('N','T') -and ont_parent.C_BASECODE LIKE 'LAB_NAME%' -- Exclude non-pcori labs -and m.MODIFIER_CD='@'; +and m.MODIFIER_CD='@' +and (m.nval_num is null or m.nval_num<=9999999) -- exclude lengths that exceed the spec +; + +execute immediate 'create index lab_result_cm_idx on lab_result_cm (PATID, ENCOUNTERID)'; +GATHER_TABLE_STATS('LAB_RESULT_CM'); END PCORNetLabResultCM; / @@ -1312,8 +780,25 @@ END PCORNetLabResultCM; create or replace procedure PCORNetHarvest as begin -INSERT INTO pmnharvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMENT_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) - VALUES('SCILHS', 'SCILHS', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, 01, 02, 1,1,2,1,2,1,2,1,2,1,1,2,2,1,1,1,2,1,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,current_date,null,current_date,null,null,null); +execute immediate 'truncate table harvest'; + +INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE) + select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 3, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.PX_DATE_MGMT, hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, + case when (select count(*) from demographic) > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, + case when (select count(*) from enrollment) > 0 then current_date else null end REFRESH_ENROLLMENT_DATE, + case when (select count(*) from encounter) > 0 then current_date else null end REFRESH_ENCOUNTER_DATE, + case when (select count(*) from diagnosis) > 0 then current_date else null end REFRESH_DIAGNOSIS_DATE, + case when (select count(*) from procedures) > 0 then current_date else null end REFRESH_PROCEDURES_DATE, + case when (select count(*) from vital) > 0 then current_date else null end REFRESH_VITAL_DATE, + case when (select count(*) from dispensing) > 0 then current_date else null end REFRESH_DISPENSING_DATE, + case when (select count(*) from lab_result_cm) > 0 then current_date else null end REFRESH_LAB_RESULT_CM_DATE, + case when (select count(*) from condition) > 0 then current_date else null end REFRESH_CONDITION_DATE, + case when (select count(*) from pro_cm) > 0 then current_date else null end REFRESH_PRO_CM_DATE, + case when (select count(*) from prescribing) > 0 then current_date else null end REFRESH_PRESCRIBING_DATE, + case when (select count(*) from pcornet_trial) > 0 then current_date else null end REFRESH_PCORNET_TRIAL_DATE, + case when (select count(*) from death) > 0 then current_date else null end REFRESH_DEATH_DATE, + case when (select count(*) from death_cause) > 0 then current_date else null end REFRESH_DEATH_CAUSE_DATE + from harvest_local hl; end PCORNetHarvest; / @@ -1321,61 +806,75 @@ end PCORNetHarvest; - - create or replace procedure PCORNetPrescribing as -sqltext varchar2(4000); begin -PMN_DROPSQL('DROP TABLE basis'); -sqltext := 'create table basis as '|| -'(select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2fact basis '|| -' inner join pmnENCOUNTER enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num '|| -' join pcornet_med basiscode '|| -' on basis.modifier_cd = basiscode.c_basecode '|| -' and basiscode.c_fullname like ''\PCORI_MOD\RX_BASIS\%'') '; -PMN_EXECUATESQL(sqltext); - -PMN_DROPSQL('DROP TABLE freq'); -sqltext := 'create table freq as '|| -'(select pcori_basecode,encounter_num,concept_cd from i2b2fact freq '|| -' inner join pmnENCOUNTER enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num '|| -' join pcornet_med freqcode '|| -' on freq.modifier_cd = freqcode.c_basecode '|| -' and freqcode.c_fullname like ''\PCORI_MOD\RX_FREQUENCY\%'') '; -PMN_EXECUATESQL(sqltext); - -PMN_DROPSQL('DROP TABLE quantity'); -sqltext := 'create table quantity as '|| -'(select nval_num,encounter_num,concept_cd from i2b2fact quantity '|| -' inner join pmnENCOUNTER enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num '|| -' join pcornet_med quantitycode '|| -' on quantity.modifier_cd = quantitycode.c_basecode '|| -' and quantitycode.c_fullname like ''\PCORI_MOD\RX_QUANTITY\'') '; - -PMN_EXECUATESQL(sqltext); +PMN_DROPSQL('drop index prescribing_idx'); +PMN_DROPSQL('drop index basis_idx'); +PMN_DROPSQL('drop index freq_idx'); +PMN_DROPSQL('drop index quantity_idx'); +PMN_DROPSQL('drop index refills_idx'); +PMN_DROPSQL('drop index supply_idx'); + +execute immediate 'truncate table prescribing'; +execute immediate 'truncate table basis'; +execute immediate 'truncate table freq'; +execute immediate 'truncate table quantity'; +execute immediate 'truncate table refills'; +execute immediate 'truncate table supply'; + +insert into basis +select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact basis + inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num + join pcornet_med basiscode + on basis.modifier_cd = basiscode.c_basecode + and basiscode.c_fullname like '\PCORI_MOD\RX_BASIS\%'; + +execute immediate 'create unique index basis_idx on basis (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; +GATHER_TABLE_STATS('BASIS'); + +insert into freq +select pcori_basecode,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact freq + inner join encounter enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num + join pcornet_med freqcode + on freq.modifier_cd = freqcode.c_basecode + and freqcode.c_fullname like '\PCORI_MOD\RX_FREQUENCY\%'; + +execute immediate 'create unique index freq_idx on freq (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; +GATHER_TABLE_STATS('FREQ'); + +insert into quantity +select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact quantity + inner join encounter enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num + join pcornet_med quantitycode + on quantity.modifier_cd = quantitycode.c_basecode + and quantitycode.c_fullname like '\PCORI_MOD\RX_QUANTITY\'; + +execute immediate 'create unique index quantity_idx on quantity (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; +GATHER_TABLE_STATS('QUANTITY'); -PMN_DROPSQL('DROP TABLE refills'); -sqltext := 'create table refills as '|| -'(select nval_num,encounter_num,concept_cd from i2b2fact refills '|| -' inner join pmnENCOUNTER enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num '|| -' join pcornet_med refillscode '|| -' on refills.modifier_cd = refillscode.c_basecode '|| -' and refillscode.c_fullname like ''\PCORI_MOD\RX_REFILLS\'') '; -PMN_EXECUATESQL(sqltext); - -PMN_DROPSQL('DROP TABLE supply'); -sqltext := 'create table supply as '|| -'(select nval_num,encounter_num,concept_cd from i2b2fact supply '|| -' inner join pmnENCOUNTER enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| -' join pcornet_med supplycode '|| -' on supply.modifier_cd = supplycode.c_basecode '|| -' and supplycode.c_fullname like ''\PCORI_MOD\RX_DAYS_SUPPLY\'') '; -PMN_EXECUATESQL(sqltext); +insert into refills +select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact refills + inner join encounter enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num + join pcornet_med refillscode + on refills.modifier_cd = refillscode.c_basecode + and refillscode.c_fullname like '\PCORI_MOD\RX_REFILLS\'; + +execute immediate 'create unique index refills_idx on refills (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; +GATHER_TABLE_STATS('REFILLS'); + +insert into supply +select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact supply + inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num + join pcornet_med supplycode + on supply.modifier_cd = supplycode.c_basecode + and supplycode.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\'; +execute immediate 'create unique index supply_idx on supply (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; +GATHER_TABLE_STATS('SUPPLY'); -- insert data with outer joins to ensure all records are included even if some data elements are missing -insert into pmnprescribing ( +insert into prescribing ( PATID ,encounterid ,RX_PROVIDERID @@ -1385,54 +884,80 @@ insert into pmnprescribing ( ,RX_END_DATE ,RXNORM_CUI --using pcornet_med pcori_cui - new column! ,RX_QUANTITY ---- modifier nval_num + ,RX_QUANTITY_UNIT ,RX_REFILLS -- modifier nval_num ,RX_DAYS_SUPPLY -- modifier nval_num ,RX_FREQUENCY --modifier with basecode lookup ,RX_BASIS --modifier with basecode lookup --- ,RAW_RX_MED_NAME, --not filling these right now + ,RAW_RX_MED_NAME -- ,RAW_RX_FREQUENCY, --- ,RAW_RXNORM_CUI + ,RAW_RXNORM_CUI ) -select distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH:MI'), m.start_date start_date, m.end_date, mo.pcori_cui - ,quantity.nval_num quantity, refills.nval_num refills, supply.nval_num supply, freq.pcori_basecode frequency, basis.pcori_basecode basis - from i2b2fact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode -inner join pmnENCOUNTER enc on enc.encounterid = m.encounter_Num +select distinct m.patient_num, m.Encounter_Num,m.provider_id, m.start_date order_date, to_char(m.start_date,'HH24:MI'), m.start_date start_date, m.end_date, mo.pcori_cui + ,quantity.nval_num quantity, 'NI' rx_quantity_unit, refills.nval_num refills, supply.nval_num supply, substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) frequency, + substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) basis + , substr(mo.c_name, 1, 50) raw_rx_med_name, substr(mo.c_basecode, 1, 50) raw_rxnorm_cui + from i2b2medfact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode +inner join encounter enc on enc.encounterid = m.encounter_Num -- TODO: This join adds several minutes to the load - must be debugged left join basis on m.encounter_num = basis.encounter_num and m.concept_cd = basis.concept_Cd + and m.start_date = basis.start_date + and m.provider_id = basis.provider_id + and m.instance_num = basis.instance_num left join freq on m.encounter_num = freq.encounter_num and m.concept_cd = freq.concept_Cd + and m.start_date = freq.start_date + and m.provider_id = freq.provider_id + and m.instance_num = freq.instance_num left join quantity on m.encounter_num = quantity.encounter_num and m.concept_cd = quantity.concept_Cd + and m.start_date = quantity.start_date + and m.provider_id = quantity.provider_id + and m.instance_num = quantity.instance_num left join refills on m.encounter_num = refills.encounter_num and m.concept_cd = refills.concept_Cd + and m.start_date = refills.start_date + and m.provider_id = refills.provider_id + and m.instance_num = refills.instance_num left join supply on m.encounter_num = supply.encounter_num and m.concept_cd = supply.concept_Cd + and m.start_date = supply.start_date + and m.provider_id = supply.provider_id + and m.instance_num = supply.instance_num + +where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); -where (basis.c_fullname is null or basis.c_fullname= '\PCORI_MOD\RX_BASIS\PR\%'); -- jgk 11/2 bugfix: filter for PR, not DI +execute immediate 'create index prescribing_idx on prescribing (PATID, ENCOUNTERID)'; +GATHER_TABLE_STATS('PRESCRIBING'); end PCORNetPrescribing; / + create or replace procedure PCORNetDispensing as -sqltext varchar2(4000); begin + +PMN_DROPSQL('drop index dispensing_idx'); + +execute immediate 'truncate table dispensing'; +/* PMN_DROPSQL('DROP TABLE supply'); sqltext := 'create table supply as '|| '(select nval_num,encounter_num,concept_cd from i2b2fact supply '|| -' inner join pmnENCOUNTER enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| +' inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| ' join pcornet_med supplycode '|| ' on supply.modifier_cd = supplycode.c_basecode '|| ' and supplycode.c_fullname like ''\PCORI_MOD\RX_DAYS_SUPPLY\'' ) '; @@ -1446,32 +971,84 @@ sqltext := 'create table amount as '|| ' on amount.modifier_cd = amountcode.c_basecode '|| ' and amountcode.c_fullname like ''\PCORI_MOD\RX_QUANTITY\'') '; PMN_EXECUATESQL(sqltext); - --- insert data with outer joins to ensure all records are included even if some data elements are missing +*/ + +/* NOTE: New transformation developed by KUMC */ -insert into pmndispensing ( +insert into dispensing ( PATID - ,PRESCRIBINGID - ,DISPENSE_DATE -- using start_date from i2b2 - ,NDC --using pcornet_med pcori_ndc - new column! - ,DISPENSE_SUP ---- modifier nval_num - ,DISPENSE_AMT -- modifier nval_num + ,PRESCRIBINGID + ,DISPENSE_DATE -- using start_date from i2b2 + ,NDC --using pcornet_med pcori_ndc - new column! + ,DISPENSE_SUP ---- modifier nval_num + ,DISPENSE_AMT -- modifier nval_num -- ,RAW_NDC ) +/* Below is the Cycle 2 fix for populating the DISPENSING table */ +with disp_status as ( + select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd + from i2b2fact ibf + join "&&i2b2_meta_schema".pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode + where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' + /* TODO: Generalize for other sites. The '< 12' makes sure only 11 digit + codes are included. */ + and length(replace(ibf.concept_cd, 'NDC:', '')) < 12 +) +, disp_quantity as ( + select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num + from i2b2fact ibf + join "&&i2b2_meta_schema".pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode + where pnm.c_fullname like '\PCORI_MOD\RX_QUANTITY\%' +) +, disp_supply as ( + select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num + from i2b2fact ibf + join "&&i2b2_meta_schema".pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode + where pnm.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\%' +) +select distinct + st.patient_num patid, + null prescribingid, + st.start_date dispense_date, + replace(st.concept_cd, 'NDC:', '') ndc, -- TODO: Generalize this for other sites. + ds.nval_num dispense_sup, + qt.nval_num dispense_amt +from disp_status st +left outer join disp_quantity qt + on st.patient_num=qt.patient_num + and st.encounter_num=qt.encounter_num + and st.concept_cd=qt.concept_cd + and st.instance_num=qt.instance_num + and st.start_date=qt.start_date +left outer join disp_supply ds + on st.patient_num=ds.patient_num + and st.encounter_num=ds.encounter_num + and st.concept_cd=ds.concept_cd + and st.instance_num=ds.instance_num + and st.start_date=ds.start_date +; + +/* NOTE: The original SCILHS transformation is below. + +-- insert data with outer joins to ensure all records are included even if some data elements are missing + select m.patient_num, null,m.start_date, NVL(mo.pcori_ndc,'NA') ,max(supply.nval_num) sup, max(amount.nval_num) amt from i2b2fact m inner join pcornet_med mo on m.concept_cd = mo.c_basecode -inner join pmnENCOUNTER enc on enc.encounterid = m.encounter_Num - +inner join encounter enc on enc.encounterid = m.encounter_Num + -- jgk bugfix 11/2 - we weren't filtering dispensing events inner join (select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2fact basis - inner join pmnENCOUNTER enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num + inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num join pcornet_med basiscode on basis.modifier_cd = basiscode.c_basecode and basiscode.c_fullname='\PCORI_MOD\RX_BASIS\DI\') basis on m.encounter_num = basis.encounter_num - and m.concept_cd = basis.concept_Cd + and m.concept_cd = basis.concept_Cd left join supply on m.encounter_num = supply.encounter_num @@ -1483,108 +1060,232 @@ inner join pmnENCOUNTER enc on enc.encounterid = m.encounter_Num and m.concept_cd = amount.concept_Cd group by m.encounter_num ,m.patient_num, m.start_date, mo.pcori_ndc; +*/ + +execute immediate 'create index dispensing_idx on dispensing (PATID)'; +GATHER_TABLE_STATS('DISPENSING'); end PCORNetDispensing; / +---------------------------------------------------------------------------------------------------------------------------------------- +---------------------------------------------------------------------------------------------------------------------------------------- +-- 11. Death - by Jeff Klann, PhD +-- Simple transform only pulls death date from patient dimension, does not rely on an ontology +-- Translated from MSSQL script by Matthew Joss +---------------------------------------------------------------------------------------------------------------------------------------- - - -create or replace PROCEDURE pcornetReport -as -i2b2pats number; -i2b2Encounters number; -i2b2facts number; -i2b2dxs number; -i2b2procs number; -i2b2lcs number; - -pmnpats number; -pmnencounters number; -pmndx number; -pmnprocs number; -pmnfacts number; -pmnenroll number; -pmnvital number; - - - -pmnlabs number; -pmnprescribings number; -pmndispensings number; -pmncond number; - - -v_runid number; +create or replace procedure PCORNetDeath as begin -select count(*) into i2b2Pats from i2b2patient; -select count(*) into i2b2Encounters from i2b2visit i inner join pmndemographic d on i.patient_num=d.patid; - - -select count(*) into pmnPats from pmndemographic; -select count(*) into pmnencounters from pmnencounter e ; -select count(*) into pmndx from pmndiagnosis; -select count(*) into pmnprocs from pmnprocedure; - -select count(*) into pmncond from pmncondition; -select count(*) into pmnenroll from pmnenrollment; -select count(*) into pmnvital from pmnvital; -select count(*) into pmnlabs from pmnlabresults_cm; -select count(*) into pmnprescribings from pmnprescribing; -select count(*) into pmndispensings from pmndispensing; - -select max(runid) into v_runid from i2pReport; -v_runid := v_runid + 1; -insert into i2pReport values( v_runid, SYSDATE, 'Pats', i2b2pats, pmnpats, i2b2pats-pmnpats); -insert into i2pReport values( v_runid, SYSDATE, 'Enrollment', i2b2pats, pmnenroll, i2b2pats-pmnpats); - -insert into i2pReport values(v_runid, SYSDATE, 'Encounters', i2b2Encounters, pmnEncounters, i2b2encounters-pmnencounters); -insert into i2pReport values(v_runid, SYSDATE, 'DX', null, pmndx, null); -insert into i2pReport values(v_runid, SYSDATE, 'PX', null, pmnprocs, null); -insert into i2pReport values(v_runid, SYSDATE, 'Condition', null, pmncond, null); -insert into i2pReport values(v_runid, SYSDATE, 'Vital', null, pmnvital, null); -insert into i2pReport values(v_runid, SYSDATE, 'Labs', null, pmnlabs, null); -insert into i2pReport values(v_runid, SYSDATE, 'Prescribing', null, pmnprescribings,null); -insert into i2pReport values(v_runid, SYSDATE, 'Dispensing', null, pmndispensings, null); - -end pcornetReport; -/ - - - + +execute immediate 'truncate table death'; + +insert into death( patid, death_date, death_date_impute, death_source, death_match_confidence) +select distinct pat.patient_num, pat.death_date, +case when vital_status_cd like 'X%' then 'B' + when vital_status_cd like 'M%' then 'D' + when vital_status_cd like 'Y%' then 'N' + else 'OT' + end death_date_impute, + 'NI' death_source, + 'NI' death_match_confidence +from ( + /* KUMC specific fix to address unknown death dates */ + select + ibp.patient_num, + case when ibf.concept_cd is not null then DATE '2100-12-31' -- in accordance with the CDM v3 spec + else ibp.death_date end death_date, + case when ibf.concept_cd is not null then 'OT' + else upper(ibp.vital_status_cd) end vital_status_cd + from i2b2patient ibp + left join i2b2fact ibf + on ibp.patient_num=ibf.patient_num + and ibf.CONCEPT_CD='DEM|VITAL:yu' +) pat +where (pat.death_date is not null or vital_status_cd like 'Z%') and pat.patient_num in (select patid from demographic); -create or replace procedure pcornetloader as -begin ----pcornetclear; -PCORNetHarvest; -PCORNetDemographic; -PCORNetEncounter; -PCORNetDiagnosis; -PCORNetCondition; -PCORNetProcedure; -PCORNetVital; -PCORNetEnroll; -PCORNetLabResultCM; -PCORNetPrescribing; -PCORNetDispensing; - -end pcornetloader; +end; / - - - - -BEGIN -pcornetloader; --- you may want to run sql statements one by one in the pcornetloader procedure :) -END; +-------------------------------------------------------------------------------- +-- PCORNetPostProc procedure +-- +-- Ideally this procedure would be empty, but as a matter of practice there are +-- often post processing cleanup tasks that needs to be complete. Such tasks +-- should reside here with the goal of incorporating them into to the proper +-- procedures above. +-------------------------------------------------------------------------------- +create or replace procedure PCORNetPostProc as +begin + + /* Copy providerid from encounter table to diagnosis, procedures tables. + CDM specification says: + "Please note: This is a field replicated from the ENCOUNTER table." + */ + merge into diagnosis d + using encounter e + on (d.encounterid = e.encounterid) + when matched then update set d.providerid = e.providerid; + + merge into procedures p + using encounter e + on (p.encounterid = e.encounterid) + when matched then update set p.providerid = e.providerid; + + merge into prescribing p + using encounter e + on (p.encounterid = e.encounterid) + when matched then update set p.rx_providerid = e.providerid; + + /* Currently in HERON, we have hight in cm and weight in oz (from visit vitals). + The CDM wants height in inches and weight in pounds. */ + update vital v set v.ht = v.ht / 2.54; + update vital v set v.wt = v.wt / 16; + + /* Remove rows from the PRESCRIBING table where RX_* fields are null + TODO: Remove this when fixed in HERON + */ + delete + from prescribing + where rx_basis is null + and rx_quantity is null + and rx_frequency is null + and rx_refills is null + ; + + /* Removed bad NDC code which make their way in from the source system + (i.e 00000000000 and 99999999999) */ + delete from dispensing + where ndc in ('00000000000', '99999999999') + ; + +end PCORNetPostProc; / -select concept "Data Type",sourceval "From i2b2",destval "In PopMedNet", diff "Difference" from i2preport where RUNID = (select max(RUNID) from I2PREPORT); - - +-------------------------------------------------------------------------------- +-- PCORNetLoader procedure +-- +-- This procedure orchestrates the execution of the procedures defined above, +-- and consists of 13 steps. Using the start_with_step parameter a step number +-- can be provided to start at any point in the sequence. This is helpful when +-- an issue is encountered during execution and restart from the beginning is +-- undesirable. +-- +-- Steps: +-- 1 - PCORNetDemographic +-- 2 - PCORNetEncounter +-- 3 - PCORNetDiagnosis +-- 4 - PCORNetCondition +-- 5 - PCORNetProcedure +-- 6 - PCORNetVital +-- 7 - PCORNetEnroll +-- 8 - PCORNetLabResultCM +-- 9 - PCORNetPrescribing +-- 10 - PCORNetDispensing +-- 11 - PCORNetDeath +-- 12 - PCORNetHarvest +-- 13 - PCORNetPostProc +-- +-------------------------------------------------------------------------------- +create or replace PROCEDURE PCORNetLoader(start_with VARCHAR2, + release_name VARCHAR2, build_num NUMBER, data_source VARCHAR2 +) AS +start_with_step int; +begin + + select step into start_with_step from ( + select 1 step, 'PCORNetDemographic' proc from dual union + select 2 step, 'PCORNetEncounter' proc from dual union + select 3 step, 'PCORNetDiagnosis' proc from dual union + select 4 step, 'PCORNetCondition' proc from dual union + select 5 step, 'PCORNetProcedure' proc from dual union + select 6 step, 'PCORNetVital' proc from dual union + select 7 step, 'PCORNetEnroll' proc from dual union + select 8 step, 'PCORNetLabResultCM' proc from dual union + select 9 step, 'PCORNetPrescribing' proc from dual union + select 10 step, 'PCORNetDispensing' proc from dual union + select 11 step, 'PCORNetDeath' proc from dual union + select 12 step, 'PCORNetHarvest' proc from dual union + select 13 step, 'PCORNetPostProc' proc from dual + ) where proc=start_with; + + if start_with_step = 1 then + LogTaskStart(release_name, 'PCORNetDemographic', build_num, data_source); + PCORNetDemographic; + LogTaskComplete(release_name, 'PCORNetDemographic', build_num, 'DEMOGRAPHIC'); + end if; + + if start_with_step <= 2 then + LogTaskStart(release_name, 'PCORNetEncounter', build_num, data_source); + PCORNetEncounter; + LogTaskComplete(release_name, 'PCORNetEncounter', build_num, 'ENCOUNTER'); + end if; + + if start_with_step <= 3 then + LogTaskStart(release_name, 'PCORNetDiagnosis', build_num, data_source); + PCORNetDiagnosis; + LogTaskComplete(release_name, 'PCORNetDiagnosis', build_num, 'DIAGNOSIS'); + end if; + + if start_with_step <= 4 then + LogTaskStart(release_name, 'PCORNetCondition', build_num, data_source); + PCORNetCondition; + LogTaskComplete(release_name, 'PCORNetCondition', build_num, 'CONDITION'); + end if; + + if start_with_step <= 5 then + LogTaskStart(release_name, 'PCORNetProcedure', build_num, data_source); + PCORNetProcedure; + LogTaskComplete(release_name, 'PCORNetProcedure', build_num, 'PROCEDURES'); + end if; + + if start_with_step <= 6 then + LogTaskStart(release_name, 'PCORNetVital', build_num, data_source); + PCORNetVital; + LogTaskComplete(release_name, 'PCORNetVital', build_num, 'VITAL'); + end if; + + if start_with_step <= 7 then + LogTaskStart(release_name, 'PCORNetEnroll', build_num, data_source); + PCORNetEnroll; + LogTaskComplete(release_name, 'PCORNetEnroll', build_num, 'ENROLLMENT'); + end if; + + if start_with_step <= 8 then + LogTaskStart(release_name, 'PCORNetLabResultCM', build_num, data_source); + PCORNetLabResultCM; + LogTaskComplete(release_name, 'PCORNetLabResultCM', build_num, 'LAB_RESULT_CM'); + end if; + + if start_with_step <= 9 then + LogTaskStart(release_name, 'PCORNetPrescribing', build_num, data_source); + PCORNetPrescribing; + LogTaskComplete(release_name, 'PCORNetPrescribing', build_num, 'PRESCRIBING'); + end if; + + if start_with_step <= 10 then + LogTaskStart(release_name, 'PCORNetDispensing', build_num, data_source); + PCORNetDispensing; + LogTaskComplete(release_name, 'PCORNetDispensing', build_num, 'DISPENSING'); + end if; + + if start_with_step <= 11 then + LogTaskStart(release_name, 'PCORNetDeath', build_num, data_source); + PCORNetDeath; + LogTaskComplete(release_name, 'PCORNetDeath', build_num, 'DEATH'); + end if; + + if start_with_step <= 12 then + LogTaskStart(release_name, 'PCORNetHarvest', build_num, data_source); + PCORNetHarvest; + LogTaskComplete(release_name, 'PCORNetHarvest', build_num, 'HARVEST'); + end if; + + PCORNetPostProc; +end PCORNetLoader; +/ \ No newline at end of file diff --git a/Oracle/PCORNetLogging.sql b/Oracle/PCORNetLogging.sql new file mode 100644 index 0000000..c4d13a1 --- /dev/null +++ b/Oracle/PCORNetLogging.sql @@ -0,0 +1,56 @@ +-------------------------------------------------------------------------------- +-- PCORNetLogging Script +-- +-- This file contains the Oracle SQL needed to create the logging table, the +-- associated sequence, and procedures for manipulating the log table. +-------------------------------------------------------------------------------- + + +whenever sqlerror continue; +drop table i2p_task_log; +whenever sqlerror exit; + +create table i2p_task_log ( + task_num number(8) not null, + release_name varchar(48) not null, + task_name varchar(48) not null, + build_num number(8) not null, + started date not null, + completed date, + num_rows number(16), + data_source varchar(48) +); + + +whenever sqlerror continue; +drop sequence i2p_task_seq; +whenever sqlerror exit; + +create sequence i2p_task_seq + minvalue 1 + start with 1 + increment by 1 + nocache +; + + +create or replace procedure LogTaskStart(r_name varchar2, t_name varchar2, b_num number, d_source varchar2) as +begin + insert into i2p_task_log (task_num, release_name, task_name, build_num, started, data_source) + values (i2p_task_seq.nextval, r_name, t_name, b_num, sysdate, d_source); +end LogTaskStart; +/ + + +create or replace procedure LogTaskComplete(r_name varchar2, t_name varchar2, b_num number, target_table varchar2) as +begin + execute immediate + 'update i2p_task_log set (completed, num_rows) = (' || + ' select sysdate, count(*) from ' || target_table || + ' )' || + 'where release_name=''' || r_name || '''' || + ' and task_name=''' || t_name || '''' || + ' and build_num=''' || b_num || '''' + ; +end LogTaskComplete; +/ \ No newline at end of file diff --git a/Oracle/backup_cdm.py b/Oracle/backup_cdm.py new file mode 100644 index 0000000..166cf1d --- /dev/null +++ b/Oracle/backup_cdm.py @@ -0,0 +1,109 @@ +''' backup_cdm - back up cdm tables that have at least one row +''' +from __future__ import print_function + +import csv +import pkg_resources as pkg +from sys import stderr + +# Allow --dry run even without cx_Oracle +try: + from cx_Oracle import DatabaseError +except ImportError: + pass + +CDM_SPEC = '../2015-06-01-PCORnet-Common-Data-Model-v3dot0-parseable-fields.csv' # noqa + +CDM3_TABLES = sorted(set(f['TABLE_NAME'] + for f in csv.DictReader( + pkg.resource_stream(__name__, CDM_SPEC)))) + + +def main(get_cursor): + cursor, backup_schema = get_cursor() + + for table in CDM3_TABLES: + bak_schema_table = ('%(backup_schema)s.%(table)s' % + dict(backup_schema=backup_schema, + table=table)) + eprint('%(table)s (to be backed up) currently has %(rcount)s rows.' + % dict(table=table, rcount=count_rows(cursor, table))) + + drop(cursor, bak_schema_table) + copy(cursor, table, bak_schema_table) + eprint('%(bak_schema_table)s now has %(rows)s rows.' % + dict(bak_schema_table=bak_schema_table, + rows=count_rows(cursor, bak_schema_table))) + + +def eprint(*args, **kwargs): + ''' ACK: http://stackoverflow.com/questions/5574702/how-to-print-to-stderr-in-python # noqa + ''' + print(*args, file=stderr, **kwargs) + + +def drop(cursor, schema_table): + eprint('Dropping %(schema_table)s (%(rows)s rows).' % + dict(schema_table=schema_table, + rows=count_rows(cursor, schema_table))) + try: + cursor.execute('drop table %(schema_table)s' % + dict(schema_table=schema_table)) + except DatabaseError, e: # noqa + # Table might not exist + pass + + +def count_rows(cursor, schema_table): + ret = None + try: + cursor.execute('select count(*) from %(schema_table)s' % + dict(schema_table=schema_table)) + ret = int(cursor.fetchall()[0][0]) + except: # noqa + pass + return ret + + +def copy(cursor, table, bak_schema_table): + eprint('Copying %(table)s to %(bak_schema_table)s.' % + dict(table=table, bak_schema_table=bak_schema_table)) + cursor.execute('create table %(bak_schema_table)s as ' + 'select * from %(table)s' % + dict(bak_schema_table=bak_schema_table, table=table)) + + +class MockCursor(object): + def __init__(self): + self.fetch_count = 1 + + def execute(self, sql): + eprint('execute: ' + sql) + + def fetchall(self): + r = [(self.fetch_count, )] + self.fetch_count += 1 + eprint('fetch returning: ' + str(r)) + return r + + +if __name__ == '__main__': + def _tcb(): + from os import environ + from sys import argv + + def get_cursor(): + host, port, sid, backup_schema = argv[1:5] + user = environ['pcornet_cdm_user'] + password = environ['pcornet_cdm'] + + if '--dry-run' in argv: + return MockCursor(), backup_schema + else: + import cx_Oracle as cx + conn = cx.connect(user, password, + dsn=cx.makedsn(host, port, sid)) + return conn.cursor(), backup_schema + + main(get_cursor) + _tcb() diff --git a/Oracle/cdm_stats.sql b/Oracle/cdm_stats.sql new file mode 100644 index 0000000..a6b903d --- /dev/null +++ b/Oracle/cdm_stats.sql @@ -0,0 +1,45 @@ +select 'DEMOGRAPHIC' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", 'N/A' "Distinct Encounters" from DEMOGRAPHIC + union all +select 'ENROLLMENT' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", 'N/A' "Distinct Encounters" from ENROLLMENT + union all +select 'ENCOUNTER' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from ENCOUNTER + union all +select 'DIAGNOSIS' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from DIAGNOSIS + union all +select 'PROCEDURES' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from PROCEDURES + union all +select 'VITAL' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from VITAL + union all +select 'DISPENSING' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", 'N/A' "Distinct Encounters" from DISPENSING + union all +select 'LAB_RESULT_CM' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from LAB_RESULT_CM + union all +select 'CONDITION' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from CONDITION + union all +select 'PRO_CM' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from PRO_CM + union all +select 'PRESCRIBING' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", to_char(count(distinct encounterid)) "Distinct Encounters" from PRESCRIBING + union all +select 'PCORNET_TRIAL' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", 'N/A' "Distinct Encounters" from PCORNET_TRIAL + union all +select 'DEATH' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", 'N/A' "Distinct Encounters" from DEATH + union all +select 'DEATH_CAUSE' "Table", to_char(count(*)) "Total Rows", to_char(count(distinct patid)) "Distinct Patients", 'N/A' "Distinct Encounters" from DEATH_CAUSE + union all +select 'HARVEST' "Table", to_char(count(*)) "Total Rows", 'N/A' "Distinct Patients", 'N/A' "Distinct Encounters" from HARVEST +; + +select sum("Size (MB)") "Total size, all tables (MB)" from ( + select + segment_name "Table", bytes/1024/1024 "Size (MB)" + from dba_segments + where owner = '&&pcornet_cdm_user' + and segment_type = 'TABLE' + and segment_name in ('DEMOGRAPHIC', 'ENROLLMENT', 'ENCOUNTER', 'DIAGNOSIS', + 'PROCEDURES', 'VITAL', 'DISPENSING', 'LAB_RESULT_CM', 'CONDITION', 'PRO_CM', + 'PRESCRIBING', 'PCORNET_TRIAL', 'DEATH', 'DEATH_CAUSE', 'HARVEST') + ); + +SELECT sum(bytes/1024/1024/1024) "Total schema size (GB)" + FROM dba_segments + WHERE owner = '&&pcornet_cdm_user'; diff --git a/Oracle/cdm_transform_tests.sql b/Oracle/cdm_transform_tests.sql new file mode 100644 index 0000000..337f9d5 --- /dev/null +++ b/Oracle/cdm_transform_tests.sql @@ -0,0 +1,667 @@ +whenever sqlerror continue +drop table test_cases; +whenever sqlerror exit; +create table test_cases ( + query_name varchar2(160) not null, + obs integer not null, + by_time integer, + by_value1 varchar2(160), + by_value2 varchar2(160), + record_n integer, + record_pct number, + distinct_patid_n integer, + pass integer not null, + description varchar2(2000) +) +; +comment on column test_cases.query_name is 'a la DRN OS SAS script query name'; +comment on column test_cases.obs is 'Obs / line number'; +comment on column test_cases.by_time is 'breakdown by year or year/month'; +comment on column test_cases.by_value1 is 'relevant nominal value'; +comment on column test_cases.by_value2 is '2nd level breakdown'; +comment on column test_cases.record_n is 'count our quanitity'; +comment on column test_cases.record_pct is 'percent of all observations'; +comment on column test_cases.distinct_patid_n is 'patient count'; +comment on column test_cases.pass is '1 for pass, 0 for fail'; + +/* Report how many condition table rows were rejected. +*/ +insert into test_cases (query_name, description, pass, obs, record_n, record_pct) +with err as ( + select count(*) qty from pcornet_cdm.ERR$_CONDITION + ), +total_condition as ( + select count(*) qty from pcornet_cdm.condition + ) +select 'rejected_condition_rows' query_name + , 'Report how many rows were rejected from the condition table due to table constraints.' description + , case when err.qty < 1000 then 1 else 0 end pass + , rownum obs + , err.qty record_n + , (err.qty / tc.qty) * 100 record_pct +from total_condition tc cross join err +; + + +/* "Low bar" test to verify that DISPENSING is not empty */ +insert into test_cases (query_name, description, obs, by_value1, by_value2, record_pct, pass) +select + 'DISPENSING' , + 'Make sure that there are at least 10 dispensing fact' description, + count(*) obs, + null by_value1, + null by_value2, + null record_pct, + case when count(*) >= 10 then 1 else 0 end pass +from dispensing +; + + +/* Test that there are at least a handful of distinct patients associated with + * DISPENSING facts. + */ +insert into test_cases (query_name, description, obs, by_value1, by_value2, record_pct, pass) +select + 'DISPENSING_PATS', + 'Make sure that there are at least 3 distinct patients with dispensing facts', + count(distinct dis.patid) obs, + null by_value1, + null by_value2, + null record_pct, + case when count(distinct dis.patid) >= 3 then 1 else 0 end pass +from dispensing dis +inner join demographic dem + on dis.patid=dem.patid +; + + +/* Test that we have some height/weight measurements +*/ +insert into test_cases (query_name, description, pass, obs, record_n, record_pct) +with total_vital as ( + select count(*) qty from pcornet_cdm.vital + ), +ht as ( + select count(*) qty from pcornet_cdm.vital + where ht is not null + ), +wt as ( + select count(*) qty from pcornet_cdm.vital + where wt is not null + ), +bmi as ( + select count(*) qty from pcornet_cdm.vital + where original_bmi is not null + ), +results as ( + select + round((ht.qty/total_vital.qty) * 100) pct_ht, + round((wt.qty/total_vital.qty) * 100) pct_wt, + round((bmi.qty/total_vital.qty) * 100) pct_bmi + from total_vital cross join ht cross join wt cross join bmi + ) +select 'some_height_measurements_4335' query_name + , 'Make sure we have at least some height records' description + , case when ht.qty > 0 then 1 else 0 end pass + , rownum obs + , ht.qty record_n + , results.pct_ht record_pct +from total_vital cross join ht cross join results +union all +select 'some_weight_measurements_4335' query_name + , 'Make sure we have at least some weight records' description + , case when wt.qty > 0 then 1 else 0 end pass + , rownum obs + , wt.qty record_n + , results.pct_wt record_pct +from total_vital cross join wt cross join results +union all +select 'some_bmi_measurements_4335' query_name + , 'Make sure we have at least some bmi records' description + , case when bmi.qty > 0 then 1 else 0 end pass + , rownum obs + , bmi.qty record_n + , results.pct_bmi record_pct +from total_vital cross join bmi cross join results +; +commit; + +/* Test to make sure we got about the same number of patients in the CDM +diagnoses that we do in i2b2. +*/ +with cdm as ( + select count(distinct patid) qty from diagnosis + ), +i2b2 as ( + select count(distinct patient_num) qty from i2b2fact + where concept_cd in ( + select concept_cd from "&&i2b2_data_schema".concept_dimension + where concept_path like '\i2b2\Diagnoses\ICD9\%' + ) + ), +diff as ( + select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct from cdm cross join i2b2 + ) +select case when diff.pct > 10 then 1/0 else 1 end diag_pat_count_ok from diff; + + +/* Test to make sure we got about the same number of patients in the CDM +procedure that we do in i2b2. +*/ +with cdm as ( + select count(distinct patid) qty from procedures + ), +i2b2 as ( + select count(distinct patient_num) qty from ( + select * from i2b2fact + where concept_cd in ( + select concept_cd from "&&i2b2_data_schema".concept_dimension + where concept_path like '\i2b2\Procedures\%' + ) + ) + ), +diff as ( + select + ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct + from cdm cross join i2b2 + ) +select case when diff.pct > 10 then 1/0 else 1 end proc_pat_count_ok from diff; + + +/* Make sure we have roughly the same number of Hispanic patients in the CDM and +i2b2. +*/ +with +num_hispanic_cdm as ( + select count(*) qty from demographic where hispanic = 'Y' + ), +num_hispanic_i2b2 as ( + select count(*) qty from "&&i2b2_data_schema".patient_dimension where ethnicity_cd = 'Y' + ), +diff as ( + select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct + from num_hispanic_cdm cdm cross join num_hispanic_i2b2 i2b2 + ) +select case when diff.pct > 10 then 1/0 else 1 end hisp_y_pat_count_ok from diff; + +-- TODO: Consider trying to combine the Y and N tests as they are copy/paste +with +num_hispanic_cdm as ( + select count(*) qty from demographic where hispanic = 'N' + ), +num_hispanic_i2b2 as ( + select count(*) qty from "&&i2b2_data_schema".patient_dimension where ethnicity_cd = 'N' + ), +diff as ( + select ((abs(cdm.qty - i2b2.qty) / i2b2.qty) * 100) pct + from num_hispanic_cdm cdm cross join num_hispanic_i2b2 i2b2 + ) +select case when diff.pct > 10 then 1/0 else 1 end hisp_n_pat_count_ok from diff; + +-- Make sure we have some diagnosis source information +select case when count(*) < 3 then 1/0 else 1 end a_few_dx_sources from ( + select distinct dx_source from diagnosis + ); + +-- Make sure we have valid DX_SOURCE values +select case when count(*) > 0 then 1/0 else 1 end valid_dx_sources from ( + select distinct dx_source from diagnosis where dx_source not in ('AD', 'DI', 'FI', 'IN', 'NI', 'UN', 'OT') + ); + +-- Make sure we have a couple principal diagnoses (PDX) +select case when count(*) < 2 then 1/0 else 1 end a_few_pdx_flags from ( + select distinct pdx from diagnosis + ); + +-- Make sure we have valid PDX values +select case when count(*) > 0 then 1/0 else 1 end valid_pdx_flags from ( + select distinct pdx from diagnosis where pdx not in ('P', 'S', 'X', 'NI', 'UN', 'OT') + ); + +-- Make sure that there are several different enrollment dates +select case when pct_distinct < 5 then 1/0 else 1 end many_enr_dates from ( + with all_enrs as ( + select count(*) qty from enrollment + ), + distinct_date as ( + select count(qty) qty from ( + select distinct enr_start_date qty from enrollment + ) + ) + select round((distinct_date.qty/all_enrs.qty) * 100, 4) pct_distinct + from distinct_date cross join all_enrs + ); + +-- Make sure most procedure dates are not null +select case when pct_not_null < 99 then 1/0 else 1 end some_px_dates_not_null from ( + with all_px as ( + select count(*) qty from procedures + ), + not_null as ( + select count(*) qty from procedures where px_date is not null + ) + select round((not_null.qty / all_px.qty) * 100, 4) pct_not_null + from not_null cross join all_px +); + +-- Make sure we have some procedure sources (px_source) +select case when count(*) = 0 then 1/0 else 1 end have_px_sources from ( + select distinct px_source from procedures where px_source is not null + ); + +-- Make sure we have some encounter types +select case when pct_known < 20 then 1/0 else 1 end some_known_enc_types from ( + with all_enc as ( + select count(*) qty from encounter + ), + known_enc as ( + select count(*) qty from encounter where enc_type is not null and enc_type != 'UN' + ) + select round((known_enc.qty / all_enc.qty) * 100, 4) pct_known + from known_enc cross join all_enc + ); + + +insert into test_cases (query_name, description, pass + , obs, record_n, record_pct, by_time, by_value1) +select 'ENC_L3_ENCTYPE_ADATE_YM' query_name + , 'TODO: chase down ~80% unknown enc_type' description + , case when enc_type in ('UN', 'OT') and pct < 25 then 1 + when enc_type not in ('UN', 'OT') and freq > 1 then 1 + else 0 end pass + , rownum obs + , t.* +from ( +with enc as ( + select extract (year from admit_date) * 100 + + extract (month from admit_date) admit_ym + , enc_type + from encounter +), enc_agg as ( + select count(*) qty from encounter +), by_time as ( + select count(*) qty, admit_ym + from enc group by admit_ym +), by_val as ( + select count(*) freq + , round(count(*) / enc_agg.qty * 100, 1) pct + , admit_ym + , enc_type + from enc cross join enc_agg + group by admit_ym, enc_type, enc_agg.qty + order by admit_ym, enc_type) +select by_val.freq + , round(by_val.freq / by_time.qty * 100, 4) pct + , by_val.admit_ym + , by_val.enc_type +from by_time join by_val on by_time.admit_ym = by_val.admit_ym +) t +; + + +insert into test_cases (query_name, description, pass + , obs, by_value1, by_value2, record_n, record_pct, distinct_patid_n) +select 'VIT_L3_SMOKING' query_name + , 'make sure we have something about patient smoking tobacco use' description + , case + when smoking = 'NI' and record_pct < 95 then 1 + when smoking != 'NI' and distinct_patid_n > 2 then 1 + else 0 + end pass + , rownum obs + , q.* +from ( +with smoking_terms as ( +select distinct pcori_basecode +from pcornet_vital +where c_fullname like '\PCORI\VITAL\TOBACCO\SMOKING\%' +), +vit as ( +select patid + , smoking + , case when smoking = 'NI' then null + when exists ( + select pcori_basecode + from smoking_terms + where smoking_terms.pcori_basecode = vital.smoking) + then null + else 1 + end value_outside +from vital +), +agg as ( + select count(*) tot from vital) +select smoking, value_outside + , count(*) + , round(count(*) / tot * 100, 4) record_pct + , count(distinct patid) distinct_patid_n +from vit cross join agg +group by smoking, value_outside, tot +order by smoking +) q +; + + +/* Test to make sure we have something about patient non-smoking tobacco use */ +insert into test_cases (query_name, description, pass + , obs, by_value1, by_value2, record_n, record_pct, distinct_patid_n) + select 'VIT_L3_TOBACCO' query_name + , 'make sure we have something about patient non-smoking tobacco use' description + , case + when tobacco = 'NI' and record_pct < 95 then 1 + when tobacco != 'NI' and distinct_patid_n > 2 then 1 + else 0 + end pass + , rownum obs + , q.* + from ( + with tobacco_terms as ( + select distinct pcori_basecode + from pcornet_vital + where c_fullname like '\PCORI\VITAL\TOBACCO\02\%' + ), + vit as ( + select patid + , tobacco + , case when tobacco = 'NI' then null + when exists ( + select pcori_basecode + from tobacco_terms + where tobacco_terms.pcori_basecode = vital.tobacco) + then null + else 1 + end value_outside + from vital + ), + agg as ( + select count(*) tot from vital) + select tobacco, value_outside + , count(*) + , round(count(*) / tot * 100, 4) record_pct + , count(distinct patid) distinct_patid_n + from vit cross join agg + group by tobacco, value_outside, tot + order by tobacco + ) q + ; + + +/* Test to make sure we have something about tobacco use types */ +insert into test_cases (query_name, description, pass + , obs, by_value1, by_value2, record_n, record_pct, distinct_patid_n) + select 'VIT_L3_TOBACCO_TYPE' query_name + , 'make sure we have something about patient tobacco use types' description + , case + when tobacco_type = 'NI' and record_pct < 95 then 1 + when tobacco_type != 'NI' and distinct_patid_n > 2 then 1 + else 0 + end pass + , rownum obs + , q.* + from ( + with tobacco_type_terms as ( + select '01' pcori_basecode from dual + union all + select '02' pcori_basecode from dual + union all + select '03' pcori_basecode from dual + union all + select '04' pcori_basecode from dual + union all + select '05' pcori_basecode from dual + union all + select '06' pcori_basecode from dual + union all + select 'NI' pcori_basecode from dual + union all + select 'UN' pcori_basecode from dual + union all + select 'OT' pcori_basecode from dual + ), + vit as ( + select patid + , tobacco_type + , case when tobacco_type = 'NI' then null + when exists ( + select pcori_basecode + from tobacco_type_terms + where tobacco_type_terms.pcori_basecode = vital.tobacco_type) + then null + else 1 + end value_outside + from vital + ), + agg as ( + select count(*) tot from vital) + select tobacco_type, value_outside + , count(*) + , round(count(*) / tot * 100, 4) record_pct + , count(distinct patid) distinct_patid_n + from vit cross join agg + group by tobacco_type, value_outside, tot + order by tobacco_type + ) q + ; + + +insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) +select 'ENC_L3_DRG' query_name + , 'make sure we have some MS-DRGs' description + , case when drg = 'NULL or missing' and record_pct < 95 then 1 + when drg != 'NULL or missing' and record_n >= 1 then 1 + else 0 + end pass + , rownum obs + , q.* +from ( +with enc as ( +select coalesce(drg, 'NULL or missing') drg +from encounter +), +agg as ( + select count(*) tot from encounter) +select drg, count(*) record_n, round(count(*) / agg.tot * 100, 4) record_pct +from enc cross join agg +group by drg, agg.tot +order by 2 desc +) q +; + + +insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) +select 'ENC_L3_N' query_name + , 'providerid: many distinct' description + , case when distinct_n / all_n * 100 >= 0.01 + and null_n / all_n * 100 < 40 then 1 + else 0 end pass + , rownum obs + , t.tag, t.distinct_n, round(t.distinct_n / t.all_n * 100, 4) +from ( +with enc as ( +select encounterid + , case when encounterid is null then 1 else null end encounterid_null + , patid + , case when patid is null then 1 else null end patid_null + , providerid + , case when providerid is null then 1 else null end providerid_null +from encounter +) +select 'encounterid' tag + , count(encounterid) all_n, count(distinct encounterid) distinct_n, count(encounterid_null) null_n +from enc +union all +select 'patid' tag + , count(patid), count(distinct patid), count(patid_null) +from enc +union all +select 'providerid' tag + , count(providerid), count(distinct providerid), count(providerid_null) +from enc +) t +; + + +insert into test_cases (query_name, description, pass, obs, by_value1, by_time, record_n, record_pct, distinct_patid_n) +select 'ENC_L3_ENCTYPE_DDATE_YM' query_name + , ' +"Discharge date. Should be populated for all +Inpatient Hospital Stay (IP) and Non-Acute +Institutional Stay (IS) encounter types." + -- PCORnet CDM v3 +' description + , case when enc_type not in ('IP', 'IS') then 1 + when discharge_date is not null then 1 + when record_pct < 10 then 1 + else 0 + end pass + , q.* +from ( + select rownum obs, q.* from ( + with enc_agg as ( + select count(*) tot from encounter + ), + enc as ( + select extract(year from discharge_date) * 100 + + extract(month from discharge_date) discharge_date + , encounterid + , patid + , enc_type + , case when discharge_date is null + or discharge_date = 'NI' + then 1 else 0 end discharge_date_null + from encounter ) + select enc_type + , discharge_date + , count(*) record_n + , round(count(*) / enc_agg.tot * 100, 4) record_pct + , count(distinct patid) distinct_patid_n + from enc cross join enc_agg + group by discharge_date, enc_type, enc_agg.tot + order by 3 desc) q + ) q +where q.obs <= 100; + + +insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) +select 'ENC_L3_DISDISP' query_name + , 'too many NI' description + , case when discharge_disposition = 'NI' and record_pct < 40 then 1 + when discharge_disposition is null and record_pct < 5 then 1 + when discharge_disposition != 'NI' then 1 + else 0 + end pass + , rownum obs + , q.* +from ( +with enc_agg as ( + select count(*) tot from encounter) +select discharge_disposition + , count(*) record_n + , round(count(*) / enc_agg.tot * 100, 4) record_pct +from encounter cross join enc_agg +group by discharge_disposition, enc_agg.tot +order by 1 +) q +; + + +insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) +select 'ENC_L3_DISSTAT' query_name + , 'too many NI' description + , case when discharge_status = 'NI' and record_pct < 40 then 1 + when discharge_status is null and record_pct < 5 then 1 + when discharge_status != 'NI' then 1 + else 0 + end pass + , rownum obs + , q.* +from ( +with enc_agg as ( + select count(*) tot from encounter) +select discharge_status + , count(*) record_n + , round(count(*) / enc_agg.tot * 100, 4) record_pct +from encounter cross join enc_agg +group by discharge_status, enc_agg.tot +order by 1 +) q +; + + +/* Should be populated for Inpatient Hospital Stay (IP) and Non-Acute +Institutional Stay (IS) encounter types. +*/ +insert into test_cases (query_name, description, pass, obs, by_value1, record_n, record_pct) +select 'ENC_L3_ADMSRC' query_name + , 'too many NI' description + , case when admitting_source = 'NI' and record_pct < 40 then 1 + when admitting_source is null and record_pct < 5 then 1 + when admitting_source != 'NI' then 1 + else 0 + end pass + , rownum obs + , q.* +from ( + with ipis as ( + select * from encounter where enc_type in ('IP', 'IS') + ), + enc_agg as ( + select count(*) tot from ipis + ) + select admitting_source + , count(*) record_n + , round(count(*) / enc_agg.tot * 100, 4) record_pct + from ipis cross join enc_agg + group by admitting_source, enc_agg.tot + order by 1 + ) q +; + +/* Due to using hostpital accounts as encounters, +we have a long tail of very long encounters; hundreds of days. + +select count(*), los from ( +select round(discharge_date - admit_date) LOS +from encounter +where enc_type in ('IP', 'IS') +) group by los +order by 2 +; +*/ + + +/* Verify that LAB_RESULT_CM contains data associated with a least half of the +labs defined in PMN_LABLOCAL. +*/ +insert into test_cases (query_name, description, obs, by_value1, by_value2, record_pct, pass) +with partial_lab_counts as ( + select lab_name, count(lab_name) lab_count + from lab_result_cm + group by lab_name +), complete_lab_counts as( + select replace(pmn.lab_name, 'LAB_NAME:', '') lab_name, + case when lc.lab_count is null then 0 else lc.lab_count end lab_count + from pmn_labnormal pmn + left outer join partial_lab_counts lc on concat('LAB_NAME:', lc.lab_name) = pmn.lab_name +), total_lab_types as ( + select count(*) num_labs from complete_lab_counts +), present_lab_types as ( + select count(*) num_labs from complete_lab_counts + where lab_count > 0 +) +select +'LAB_RESULT_CM Test' query_name, +'LAB_RESULT_CM contains data associated with at least half of the defined labs' description, +rownum obs, +plt.num_labs by_value1, +tlt.num_labs by_values2, +round(plt.num_labs/tlt.num_labs, 4) record_pct, +case when (plt.num_labs/tlt.num_labs) > 0.5 then 1 else 0 end pass +from present_lab_types plt +cross join total_lab_types tlt +; + +select case when count(*) > 0 then 1/0 else 1 end all_test_cases_pass from ( +select * from test_cases where pass = 0 +); + diff --git a/Oracle/condition.sql b/Oracle/condition.sql new file mode 100644 index 0000000..f281899 --- /dev/null +++ b/Oracle/condition.sql @@ -0,0 +1,109 @@ +/** condition - create and populate the condition table. +*/ +BEGIN +PMN_DROPSQL('DROP TABLE condition'); +END; +/ +CREATE TABLE condition( + CONDITIONID varchar(19) primary key, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, + REPORT_DATE date NULL, + RESOLVE_DATE date NULL, + ONSET_DATE date NULL, + CONDITION_STATUS varchar(2) NULL, + CONDITION varchar(18) NOT NULL, + CONDITION_TYPE varchar(2) NOT NULL, + CONDITION_SOURCE varchar(2) NOT NULL, + RAW_CONDITION_STATUS varchar(2) NULL, + RAW_CONDITION varchar(18) NULL, + RAW_CONDITION_TYPE varchar(2) NULL, + RAW_CONDITION_SOURCE varchar(2) NULL +) +/ + +BEGIN +PMN_DROPSQL('DROP sequence condition_seq'); +END; +/ +create sequence condition_seq +/ + +create or replace trigger condition_trg +before insert on condition +for each row +begin + select condition_seq.nextval into :new.CONDITIONID from dual; +end; +/ + +BEGIN +PMN_DROPSQL('DROP TABLE sourcefact2'); +END; +/ + +CREATE TABLE SOURCEFACT2 ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + DXSOURCE VARCHAR2(50) NULL, + C_FULLNAME VARCHAR2(700) NOT NULL + ) +/ +create or replace procedure PCORNetCondition as +begin + +PMN_DROPSQL('drop index condition_idx'); +PMN_DROPSQL('drop index sourcefact2_idx'); + +execute immediate 'truncate table condition'; +execute immediate 'truncate table sourcefact2'; + +insert into sourcefact2 + select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname + from i2b2fact factline + inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + where dxsource.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\%'; + +execute immediate 'create index sourcefact2_idx on sourcefact2 (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('SOURCEFACT2'); + +/*create_error_table('CONDITION');*/ + +insert into condition (patid, encounterid, report_date, resolve_date, condition, condition_type, condition_status, condition_source) +select distinct factline.patient_num, min(factline.encounter_num) encounterid, min(factline.start_date) report_date, NVL(max(factline.end_date),null) resolve_date, diag.pcori_basecode, +SUBSTR(diag.c_fullname,18,2) condition_type, + NVL2(max(factline.end_date) , 'RS', 'NI') condition_status, -- Imputed so might not be entirely accurate + NVL(SUBSTR(max(dxsource),INSTR(max(dxsource), ':')+1,2),'NI') condition_source +from i2b2fact factline +inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num +inner join pcornet_diag diag on diag.c_basecode = factline.concept_cd + left outer join sourcefact2 sf +on factline.patient_num=sf.patient_num +and factline.encounter_num=sf.encounter_num +and factline.provider_id=sf.provider_id +and factline.concept_cd=sf.concept_Cd +and factline.start_date=sf.start_Date +where diag.c_fullname like '\PCORI\DIAGNOSIS\%' +and sf.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\CONDITION_SOURCE\%' +group by factline.patient_num, diag.pcori_basecode, diag.c_fullname +/*log errors into ERR$_CONDITION reject limit unlimited*/ +; + +execute immediate 'create index condition_idx on condition (PATID, ENCOUNTERID)'; +GATHER_TABLE_STATS('CONDITION'); + +end PCORNetCondition; +/ +BEGIN +PCORNetCondition(); +END; +/ +update cdm_status +set end_time = sysdate, records = (select count(*) from condition) +where task = 'condition' +/ +select records from cdm_status where task = 'condition' diff --git a/Oracle/death.sql b/Oracle/death.sql new file mode 100644 index 0000000..9c2aaeb --- /dev/null +++ b/Oracle/death.sql @@ -0,0 +1,54 @@ +/** death - create and populate the death table. +*/ +insert into cdm_status (task, start_time) select 'death', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE death'); +END; +/ +CREATE TABLE death( + PATID varchar(50) NOT NULL, + DEATH_DATE date NOT NULL, + DEATH_DATE_IMPUTE varchar(2) NULL, + DEATH_SOURCE varchar(2) NOT NULL, + DEATH_MATCH_CONFIDENCE varchar(2) NULL +) +/ +create or replace procedure PCORNetDeath as +begin + +execute immediate 'truncate table death'; + +insert into death(patid, death_date, death_date_impute, death_source, death_match_confidence) +with death_source_map as ( + select 'DEM|VITAL:y' concept_cd, 'L' death_source, null unknown from dual + union all + select 'DEM|VITAL:yu' concept_cd, 'L' death_source, 1 unknown from dual + union all + select 'DEM|VITAL|SSA:y' concept_cd, 'D' death_source, null unknown from dual + union all + select 'NAACCR|1760:0' concept_cd, 'T' death_source, null unknown from dual + union all + select 'NAACCR|1760:4' concept_cd, 'T' death_source, null unknown from dual + -- Possible exception, NAACCR|1760:12 is not handled here as there are no examples in the data. + -- This is the case where date of death is flagged as unknown. +) +select distinct /*+ parallel(0) */ obs.patient_num patid + , case when dmap.unknown = 1 then DATE '2100-12-31' else obs.start_date end death_date + , case when dmap.unknown = 1 then 'OT' else 'N' end death_date_impute + , dmap.death_source + , 'NI' death_match_confidence +from pcornet_cdm.i2b2fact obs +join death_source_map dmap on obs.concept_cd = dmap.concept_cd; + +end; +/ +BEGIN +PCORNetDeath(); +END; +/ +update cdm_status +set end_time = sysdate, records = (select count(*) from death) +where task = 'death' +/ +select records from cdm_status where task = 'death' \ No newline at end of file diff --git a/Oracle/death_cause.sql b/Oracle/death_cause.sql new file mode 100644 index 0000000..c3f5df7 --- /dev/null +++ b/Oracle/death_cause.sql @@ -0,0 +1,22 @@ +/** death_cause - create the death_cause table. +*/ +insert into cdm_status (task, start_time) select 'death_cause', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE death_cause'); +END; +/ +CREATE TABLE death_cause( + PATID varchar(50) NOT NULL, + DEATH_CAUSE varchar(8) NOT NULL, + DEATH_CAUSE_CODE varchar(2) NOT NULL, + DEATH_CAUSE_TYPE varchar(2) NOT NULL, + DEATH_CAUSE_SOURCE varchar(2) NOT NULL, + DEATH_CAUSE_CONFIDENCE varchar(2) NULL +) +/ +update cdm_status +set end_time = sysdate, records = (select count(*) from death_cause) +where task = 'death_cause' +/ +select records + 1 from cdm_status where task = 'death_cause' \ No newline at end of file diff --git a/Oracle/demographic.sql b/Oracle/demographic.sql new file mode 100644 index 0000000..beab9ae --- /dev/null +++ b/Oracle/demographic.sql @@ -0,0 +1,222 @@ +/** demographic - create and populate the demographic table. +*/ +insert into cdm_status (task, start_time) select 'demographic', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE demographic'); +END; +/ +CREATE TABLE demographic( + PATID varchar(50) NOT NULL, + BIRTH_DATE date NULL, + BIRTH_TIME varchar(5) NULL, + SEX varchar(2) NULL, + SEXUAL_ORIENTATION varchar(2) NULL, + GENDER_IDENTITY varchar(2) NULL, + HISPANIC varchar(2) NULL, + BIOBANK_FLAG varchar(1) DEFAULT 'N', + RACE varchar(2) NULL, + PAT_PREF_LANGUAGE_SPOKEN varchar(3) NULL, + RAW_SEX varchar(50) NULL, + RAW_SEXUAL_ORIENTATION varchar(50) NULL, + RAW_GENDER_IDENTITY varchar(50) NULL, + RAW_HISPANIC varchar(50) NULL, + RAW_RACE varchar(50) NULL, + RAW_PAT_PREF_LANGUAGE_SPOKEN varchar(50) NULL +) +/ +create or replace procedure PCORNetDemographic as + +sqltext varchar2(4000); +cursor getsql is +--1 -- S,R,NH + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''1'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + ''''||sex.pcori_basecode||''','|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + ''''||race.pcori_basecode||''''|| + ' from i2b2patient p '|| + ' where lower(p.sex_cd) in ('||lower(sex.c_dimcode)||') '|| + ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| + ' and lower(nvl(p.ethnicity_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') ' + from pcornet_demo race, pcornet_demo sex + where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' + and race.c_visualattributes like 'L%' + and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' + and sex.c_visualattributes like 'L%' +union -- A - S,R,H +select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''A'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + ''''||sex.pcori_basecode||''','|| + '''NI'','|| + '''NI'','|| + ''''||hisp.pcori_basecode||''','|| + ''''||race.pcori_basecode||''''|| + ' from i2b2patient p '|| + ' where lower(p.sex_cd) in ('||lower(sex.c_dimcode)||') '|| + ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| + ' and lower(p.ethnicity_cd) in ('||lower(hisp.c_dimcode)||') ' + from pcornet_demo race, pcornet_demo hisp, pcornet_demo sex + where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' + and race.c_visualattributes like 'L%' + and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' + and sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' + and sex.c_visualattributes like 'L%' +union --2 S, nR, nH + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''2'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + ''''||sex.pcori_basecode||''','|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + '''NI'''|| + ' from i2b2patient p '|| + ' where lower(nvl(p.sex_cd,''xx'')) in ('||lower(sex.c_dimcode)||') '|| + ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') '|| + ' and lower(nvl(p.ethnicity_cd,''ni'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') ' + from pcornet_demo sex + where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' + and sex.c_visualattributes like 'L%' +union --3 -- nS,R, NH + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''3'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + ''''||race.pcori_basecode||''''|| + ' from i2b2patient p '|| + ' where lower(nvl(p.sex_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''SEX'') '|| + ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| + ' and lower(nvl(p.ethnicity_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'')' + from pcornet_demo race + where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' + and race.c_visualattributes like 'L%' +union --B -- nS,R, H + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''B'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + ''''||hisp.pcori_basecode||''','|| + ''''||race.pcori_basecode||''''|| + ' from i2b2patient p '|| + ' where lower(nvl(p.sex_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''SEX'') '|| + ' and lower(p.race_cd) in ('||lower(race.c_dimcode)||') '|| + ' and lower(p.ethnicity_cd) in ('||lower(hisp.c_dimcode)||') ' + from pcornet_demo race,pcornet_demo hisp + where race.c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' + and race.c_visualattributes like 'L%' + and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' +union --4 -- S, NR, H + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''4'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + ''''||sex.pcori_basecode||''','|| + '''NI'','|| + '''NI'','|| + ''''||hisp.pcori_basecode||''','|| + '''NI'''|| + ' from i2b2patient p '|| + ' where lower(nvl(p.sex_cd,''NI'')) in ('||lower(sex.c_dimcode)||') '|| + ' and lower(nvl(p.ethnicity_cd,''NI'')) in ('||lower(hisp.c_dimcode)||') '|| + ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') ' + from pcornet_demo sex, pcornet_demo hisp + where sex.c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' + and sex.c_visualattributes like 'L%' + and hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' +union --5 -- NS, NR, H + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''5'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + ''''||hisp.pcori_basecode||''','|| + '''NI'''|| + ' from i2b2patient p '|| + ' where lower(nvl(p.sex_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''SEX'') '|| + ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') '|| + ' and lower(nvl(p.ethnicity_cd,''NI'')) in ('||lower(hisp.c_dimcode)||') ' + from pcornet_demo hisp + where hisp.c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%' + and hisp.c_visualattributes like 'L%' +union --6 -- NS, NR, nH + select 'insert into demographic(raw_sex,PATID, BIRTH_DATE, BIRTH_TIME,SEX, SEXUAL_ORIENTATION, GENDER_IDENTITY, HISPANIC, RACE) '|| + ' select ''6'',patient_num, '|| + ' birth_date, '|| + ' to_char(birth_date,''HH24:MI''), '|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + '''NI'','|| + '''NI'''|| + ' from i2b2patient p '|| + ' where lower(nvl(p.sex_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''SEX'') '|| + ' and lower(nvl(p.ethnicity_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''HISPANIC'') '|| + ' and lower(nvl(p.race_cd,''xx'')) not in (select lower(code) from pcornet_codelist where codetype=''RACE'') ' + from dual; + +begin +pcornet_popcodelist; + +PMN_DROPSQL('drop index demographic_pk'); + +execute immediate 'truncate table demographic'; + +OPEN getsql; +LOOP +FETCH getsql INTO sqltext; + EXIT WHEN getsql%NOTFOUND; +-- insert into st values (sqltext); + execute immediate sqltext; + COMMIT; +END LOOP; +CLOSE getsql; + +-- Logic of the using statment is - +-- When the source (i2b2patient) doesn't provide a lanaguage_cd, substitute 'no information', which will be coded +-- as 'NI'. When the source provides a language_cd that isn't in the mapping table, substitute the code 'OT'. +merge into demographic d +using ( + select NVL(code, 'OT') as code, language_cd, patient_num + from language_code + right join i2b2patient on + case when language_cd is NULL then 'no information' else language_cd end = lower(descriptive_text) +) l +on (d.patid = l.patient_num) +when matched then update +set d.PAT_PREF_LANGUAGE_SPOKEN = l.code, +d.RAW_PAT_PREF_LANGUAGE_SPOKEN = l.language_cd; + +execute immediate 'create unique index demographic_pk on demographic (PATID)'; +GATHER_TABLE_STATS('DEMOGRAPHIC'); + +end PCORNetDemographic; +/ +BEGIN +PCORNetDemographic(); +END; +/ +update cdm_status +set end_time = sysdate, records = (select count(*) from demographic) +where task = 'demographic' +/ +select records from cdm_status where task = 'demographic' \ No newline at end of file diff --git a/Oracle/diagnosis.sql b/Oracle/diagnosis.sql new file mode 100644 index 0000000..166767d --- /dev/null +++ b/Oracle/diagnosis.sql @@ -0,0 +1,310 @@ +/** diagnosis - create and populate the diagnosis table. +*/ +BEGIN +PMN_DROPSQL('DROP TABLE diagnosis'); +END; +/ +CREATE TABLE diagnosis( + DIAGNOSISID varchar(19) primary key, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NOT NULL, + ENC_TYPE varchar(2) NULL, + ADMIT_DATE date NULL, + DX_DATE date NULL, + PROVIDERID varchar(50) NULL, + DX varchar(18) NOT NULL, + DX_TYPE varchar(2) NOT NULL, + DX_SOURCE varchar(2) NOT NULL, + DX_ORIGIN varchar(2) NULL, + PDX varchar(2) NULL, + DX_POA varchar(2) NULL, + RAW_DX varchar(50) NULL, + RAW_DX_TYPE varchar(50) NULL, + RAW_DX_SOURCE varchar(50) NULL, + RAW_ORIGDX varchar(50) NULL, + RAW_PDX varchar(50) NULL, + RAW_DX_POA varchar(50) NULL +) +/ + +BEGIN +PMN_DROPSQL('DROP sequence diagnosis_seq'); +END; +/ +create sequence diagnosis_seq +/ + +create or replace trigger diagnosis_trg +before insert on diagnosis +for each row +begin + select diagnosis_seq.nextval into :new.DIAGNOSISID from dual; +end; +/ + +BEGIN +PMN_DROPSQL('DROP TABLE sourcefact'); +END; +/ + +CREATE TABLE SOURCEFACT ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + DXSOURCE VARCHAR2(50) NULL, + C_FULLNAME VARCHAR2(700) NOT NULL + ) +/ + +BEGIN +PMN_DROPSQL('DROP TABLE pdxfact'); +END; +/ + +CREATE TABLE PDXFACT ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + PDXSOURCE VARCHAR2(50) NULL, + C_FULLNAME VARCHAR2(700) NOT NULL + ) +/ + +BEGIN +PMN_DROPSQL('DROP TABLE originfact'); +END; +/ + +CREATE TABLE ORIGINFACT ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + ORIGINSOURCE VARCHAR2(50) NULL, + C_FULLNAME VARCHAR2(700) NOT NULL + ) +/ + +BEGIN +PMN_DROPSQL('DROP TABLE poafact'); +END; +/ + +CREATE TABLE POAFACT ( + PATIENT_NUM NUMBER(38) NOT NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + PROVIDER_ID VARCHAR2(50) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL, + START_DATE DATE NOT NULL, + POASOURCE VARCHAR2(50) NULL, + RAWPOASOURCE VARCHAR2(50) NULL, + C_FULLNAME VARCHAR2(700) NOT NULL + ) +/ + +create or replace procedure PCORNetDiagnosis as +begin + +PMN_DROPSQL('drop index diagnosis_idx'); +PMN_DROPSQL('drop index sourcefact_idx'); +PMN_DROPSQL('drop index pdxfact_idx'); +PMN_DROPSQL('drop index originfact_idx'); +PMN_DROPSQL('drop index poafact_idx'); + +execute immediate 'truncate table diagnosis'; +execute immediate 'truncate table sourcefact'; +execute immediate 'truncate table pdxfact'; +execute immediate 'truncate table originfact'; +execute immediate 'truncate table poafact'; + +insert into sourcefact + select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode dxsource, dxsource.c_fullname + from i2b2fact factline + inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + where dxsource.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\%'; + +execute immediate 'create index sourcefact_idx on sourcefact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('SOURCEFACT'); + +insert into pdxfact + select distinct patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode pdxsource,dxsource.c_fullname + from i2b2fact factline + inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + and dxsource.c_fullname like '\PCORI_MOD\PDX\%'; + +execute immediate 'create index pdxfact_idx on pdxfact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('PDXFACT'); + +insert into originfact --CDM 3.1 addition + select patient_num, encounter_num, provider_id, concept_cd, start_date, dxsource.pcori_basecode originsource, dxsource.c_fullname + from i2b2fact factline + inner join ENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd =dxsource.c_basecode + and dxsource.c_fullname like '\PCORI_MOD\DX_ORIGIN\%'; + +execute immediate 'create index originfact_idx on originfact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('ORIGINFACT'); + +insert into poafact + select patient_num, encounter_num, provider_id, concept_cd, start_date, 'Y' poasource, 'Yes' rawpoasource, dxsource.c_fullname + from i2b2fact factline + inner join ENCOUNTER enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + inner join pcornet_diag dxsource on factline.modifier_cd = dxsource.c_basecode + and dxsource.c_fullname like '\PCORI_MOD\DX_ORIGIN\BI\DX|BILL:POA\%' + + union all + + select patient_num, factline.encounter_num, provider_id, concept_cd, start_date, + case + when sf.tval_char = 'No' then 'N' + when sf.tval_char = 'Unknown' then 'UN' + when sf.tval_char = 'Clinically Undetermined' then 'W' + when sf.tval_char = 'Exempt from POA reporting' then '1' + else 'OT' + end poasource, + sf.tval_char rawpoasource, '@' + from i2b2fact factline + join &&i2b2_data_schema.supplemental_fact sf on factline.instance_num = sf.instance_num and sf.source_column = 'POA_ALT'; + +execute immediate 'create index poafact_idx on poafact (patient_num, encounter_num, provider_id, concept_cd, start_date)'; +GATHER_TABLE_STATS('POAFACT'); + +insert into diagnosis (patid, encounterid, enc_type, admit_date, dx_date, providerid, dx, dx_type, dx_source, dx_origin, pdx, dx_poa, raw_dx_poa) +/* KUMC started billing with ICD10 on Oct 1, 2015. */ +with icd10_transition as ( + select date '2015-10-01' as cutoff from dual +) + +/* Encoding of DX type from the CDM 3.1 Specification + http://pcornet.org/wp-content/uploads/2016/11/2016-11-15-PCORnet-Common-Data-Model-v3.1_Specification.pdf + */ +, dx_type as ( + select '09' as icd_9_cm + , '10' as icd_10_cm + , '11' as icd_11_cm + , 'SM' as snomed_ct + , 'NI' as no_info + , 'UN' as unknown + , 'OT' as other + from dual) + +/* DX_IDs may have mappings to both ICD9 and ICD10 */ +, has9 as ( + select distinct c_basecode, pcori_basecode icd9_code + from "&&i2b2_meta_schema".pcornet_diag diag + where diag.c_fullname like '\PCORI\DIAGNOSIS\09%' + and pcori_basecode is not null +) +, has10 as ( + select distinct c_basecode, pcori_basecode icd10_code + from "&&i2b2_meta_schema".pcornet_diag diag + where diag.c_fullname like '\PCORI\DIAGNOSIS\10%' + and pcori_basecode is not null +) +, diag as ( + select distinct diag.c_basecode, diag.pcori_basecode, has9.icd9_code, has10.icd10_code + , case when diag.pcori_basecode = has9.icd9_code then (select icd_9_cm from dx_type) + when diag.pcori_basecode = has10.icd10_code then (select icd_10_cm from dx_type) + else (select no_info from dx_type) + end dx_type + from pcornet_diag diag + left join has9 on has9.c_basecode = diag.c_basecode + left join has10 on has10.c_basecode = diag.c_basecode + + where diag.c_fullname like '\PCORI\DIAGNOSIS\%' + and diag.pcori_basecode is not null +) +/* Convert i2b2 diagnosis facts to pcori diagnosis facts + * Note: The mapping of i2b2 diagnosis facts to pcori diagnosis facts + * with respect to ICD9 and ICD10 is not 1-to-1 (there may exist an i2b2 + * diagnosis fact that could be represented as both ICD9 and ICD10 pcori fact) + * + * To prevent an increase in the number of post conversion facts, prioritize + * filtering ICD10s prior to 1/1/2015 and ICD9s after 1/1/2015 if resulting + * facts produce duplicates based on the i2b2 fact primary key. + */ +, diag_fact_merge as ( + select factline.*, diag.* + , row_number() over (partition by factline.patient_num + , factline.encounter_num + , factline.concept_cd + , factline.start_date + , factline.modifier_cd + , factline.instance_num + order by (case when diag.dx_type = (select icd_9_cm from dx_type) + and factline.start_date >= (select cutoff from icd10_transition) then 0 + when diag.dx_type = (select icd_10_cm from dx_type) + and factline.start_date < (select cutoff from icd10_transition) then 0 + else 1 + end) asc) unique_row + from i2b2fact factline + join diag on diag.c_basecode = factline.concept_cd +) +, diag_fact_cutoff_filter as ( + select * from diag_fact_merge where unique_row = 1 +) + +select distinct factline.patient_num, factline.encounter_num encounterid, enc_type, enc.admit_date, factline.start_date, enc.providerid + , factline.pcori_basecode dx + , factline.dx_type dxtype, + CASE WHEN enc_type='AV' THEN 'FI' ELSE nvl(SUBSTR(dxsource,INSTR(dxsource,':')+1,2) ,'NI') END dx_source, + nvl(SUBSTR(originsource,INSTR(originsource, ':')+1,2),'NI') dx_origin, + CASE WHEN enc_type in ('EI', 'IP', 'IS', 'OS') + THEN nvl(SUBSTR(pdxsource,INSTR(pdxsource, ':')+1,2),'NI') + ELSE null END PDX, + CASE WHEN enc_type in ('EI', 'IP') + THEN nvl(SUBSTR(poasource,INSTR(poasource, ':')+1,2),'NI') + ELSE null END DX_POA + , rawpoasource RAW_DX_POA +from diag_fact_cutoff_filter factline +inner join encounter enc on enc.patid = factline.patient_num and enc.encounterid = factline.encounter_Num + left outer join sourcefact +on factline.patient_num=sourcefact.patient_num +and factline.encounter_num=sourcefact.encounter_num +and factline.provider_id=sourcefact.provider_id +and factline.concept_cd=sourcefact.concept_Cd +and factline.start_date=sourcefact.start_Date +left outer join pdxfact +on factline.patient_num=pdxfact.patient_num +and factline.encounter_num=pdxfact.encounter_num +and factline.provider_id=pdxfact.provider_id +and factline.concept_cd=pdxfact.concept_cd +and factline.start_date=pdxfact.start_Date +left outer join originfact --CDM 3.1 addition +on factline.patient_num=originfact.patient_num +and factline.encounter_num=originfact.encounter_num +and factline.provider_id=originfact.provider_id +and factline.concept_cd=originfact.concept_cd +and factline.start_date=originfact.start_Date +left outer join poafact +on factline.patient_num=poafact.patient_num +and factline.encounter_num=poafact.encounter_num +and factline.provider_id=poafact.provider_id +and factline.concept_cd=poafact.concept_cd +and factline.start_date=poafact.start_Date + +where (sourcefact.c_fullname like '\PCORI_MOD\CONDITION_OR_DX\DX_SOURCE\%' or sourcefact.c_fullname is null) +-- order by enc.admit_date desc +; + +execute immediate 'create index diagnosis_idx on diagnosis (PATID, ENCOUNTERID)'; +GATHER_TABLE_STATS('DIAGNOSIS'); + +end PCORNetDiagnosis; +/ +BEGIN +PCORNetDiagnosis(); +END; +/ +update cdm_status +set end_time = sysdate, records = (select count(*) from diagnosis) +where task = 'diagnosis' +/ +select records from cdm_status where task = 'diagnosis' \ No newline at end of file diff --git a/Oracle/dispensing.sql b/Oracle/dispensing.sql new file mode 100644 index 0000000..c66f5dc --- /dev/null +++ b/Oracle/dispensing.sql @@ -0,0 +1,243 @@ +/** dispensing - create and populate the dispensing table. +*/ +insert into cdm_status (task, start_time) select 'dispensing', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE dispensing'); +END; +/ +CREATE TABLE dispensing( + DISPENSINGID varchar(19) primary key, + PATID varchar(50) NOT NULL, + PRESCRIBINGID varchar(19) NULL, + DISPENSE_DATE date NOT NULL, + NDC varchar (11) NOT NULL, + DISPENSE_SUP number(18) NULL, + DISPENSE_AMT number(18) NULL, + DISPENSE_DOSE_DISP number(18) NULL, + DISPENSE_DOSE_DISP_UNIT varchar(50) NULL, + DISPENSE_ROUTE varchar(50) NULL, + RAW_NDC varchar (50) NULL, + RAW_DISPENSE_DOSE_DISP varchar(50) NULL, + RAW_DISPENSE_DOSE_DISP_UNIT varchar(50) NULL, + RAW_DISPENSE_ROUTE varchar(50) NULL, + DISPENSE_SOURCE varchar(2) NULL +) +/ + +BEGIN +PMN_DROPSQL('DROP sequence dispensing_seq'); +END; +/ +create sequence dispensing_seq +/ + +create or replace trigger dispensing_trg +before insert on dispensing +for each row +begin + select dispensing_seq.nextval into :new.DISPENSINGID from dual; +end; +/ + +BEGIN +PMN_DROPSQL('DROP TABLE DISP_SUPPLY'); --Changed the table 'supply' to the name 'disp_supply' to avoid conflicts with the prescribing procedure, Matthew Joss 8/16/16 +END; +/ + +CREATE TABLE DISP_SUPPLY ( + NVAL_NUM NUMBER(18,5) NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL + ) +/ + +BEGIN +PMN_DROPSQL('DROP TABLE amount'); +END; +/ + +CREATE TABLE AMOUNT ( + NVAL_NUM NUMBER(18,5) NULL, + ENCOUNTER_NUM NUMBER(38) NOT NULL, + CONCEPT_CD VARCHAR2(50) NOT NULL + ) +/ +create or replace procedure PCORNetDispensing as +begin + +PMN_DROPSQL('drop index dispensing_idx'); + +execute immediate 'truncate table dispensing'; +/* +PMN_DROPSQL('DROP TABLE supply'); +sqltext := 'create table supply as '|| +'(select nval_num,encounter_num,concept_cd from i2b2fact supply '|| +' inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| +' join pcornet_med supplycode '|| +' on supply.modifier_cd = supplycode.c_basecode '|| +' and supplycode.c_fullname like ''\PCORI_MOD\RX_DAYS_SUPPLY\'' ) '; +PMN_EXECUATESQL(sqltext); + + +PMN_DROPSQL('DROP TABLE amount'); +sqltext := 'create table amount as '|| +'(select nval_num,encounter_num,concept_cd from i2b2fact amount '|| +' join pcornet_med amountcode '|| +' on amount.modifier_cd = amountcode.c_basecode '|| +' and amountcode.c_fullname like ''\PCORI_MOD\RX_QUANTITY\'') '; +PMN_EXECUATESQL(sqltext); +*/ + +/* NOTE: New transformation developed by KUMC */ + +insert into dispensing ( + PATID + ,PRESCRIBINGID + ,DISPENSE_DATE -- using start_date from i2b2 + ,NDC --using pcornet_med pcori_ndc - new column! + ,DISPENSE_SUP ---- modifier nval_num + ,DISPENSE_AMT -- modifier nval_num + ,DISPENSE_DOSE_DISP + ,DISPENSE_DOSE_DISP_UNIT + ,DISPENSE_ROUTE + ,RAW_NDC + ,RAW_DISPENSE_DOSE_DISP + ,RAW_DISPENSE_DOSE_DISP_UNIT + ,RAW_DISPENSE_ROUTE + ,DISPENSE_SOURCE +) +/* Below is the Cycle 2 fix for populating the DISPENSING table */ +with disp_status as ( + select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd + from i2b2fact ibf + join "&&i2b2_meta_schema".pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode + where pnm.c_fullname like '\PCORI_MOD\RX_BASIS\DI\%' + /* TODO: Generalize for other sites. The '< 12' makes sure only 11 digit + codes are included. */ + and length(replace(ibf.concept_cd, 'NDC:', '')) < 12 +) +, disp_quantity as ( + select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num + from i2b2fact ibf + join "&&i2b2_meta_schema".pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode + where pnm.c_fullname like '\PCORI_MOD\RX_QUANTITY\%' +) +, disp_supply as ( + select ibf.patient_num, ibf.encounter_num, ibf.concept_cd, ibf.instance_num, ibf.start_date, ibf.modifier_cd, ibf.nval_num + from i2b2fact ibf + join "&&i2b2_meta_schema".pcornet_med pnm + on ibf.modifier_cd=pnm.c_basecode + where pnm.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\%' +) +, disp_dose as ( + select encounter_num + , instance_num + , to_number(tval_char) dose + from &&i2b2_data_schema.supplemental_fact + where source_column = 'DISCRETE_DOSE' +) +, disp_unit as ( + select sf.encounter_num + , sf.instance_num + , nvl(um.code, 'OT') code + , sf.tval_char + from &&i2b2_data_schema.supplemental_fact sf + left join unit_map um on sf.tval_char = um.unit_name + where sf.source_column = 'DOSE_UNITS' +) +, disp_route as ( + select sf.encounter_num + , sf.instance_num + , rm.code + , sf.tval_char + from &&i2b2_data_schema.supplemental_fact sf + left join route_map rm on lower(sf.tval_char) = lower(rm.route_name) + where sf.source_column = 'ADMIN_ROUTE' +) +select distinct + st.patient_num patid, + null prescribingid, + st.start_date dispense_date, + replace(st.concept_cd, 'NDC:', '') ndc, -- TODO: Generalize this for other sites. + ds.nval_num dispense_sup, + qt.nval_num dispense_amt, + dd.dose dispense_dose_disp, + du.code dispense_dose_disp_unit, + dr.code dispense_route, + null raw_ndc, + dd.dose raw_dispense_dose_disp, + du.tval_char raw_dispense_dose_disp_unit, + dr.tval_char raw_dispense_route, + 'PM' dispense_source +from disp_status st +left outer join disp_quantity qt + on st.patient_num=qt.patient_num + and st.encounter_num=qt.encounter_num + and st.concept_cd=qt.concept_cd + and st.instance_num=qt.instance_num + and st.start_date=qt.start_date +left outer join disp_supply ds + on st.patient_num=ds.patient_num + and st.encounter_num=ds.encounter_num + and st.concept_cd=ds.concept_cd + and st.instance_num=ds.instance_num + and st.start_date=ds.start_date +left outer join disp_dose dd + on st.encounter_num = dd.encounter_num + and st.instance_num = dd.instance_num +left outer join disp_unit du + on st.encounter_num = du.encounter_num + and st.instance_num = du.instance_num +left outer join disp_route dr + on st.encounter_num = dr.encounter_num + and st.instance_num = dr.instance_num +; + +/* NOTE: The original SCILHS transformation is below. + +-- insert data with outer joins to ensure all records are included even if some data elements are missing + +select m.patient_num, null,m.start_date, NVL(mo.pcori_ndc,'NA') + ,max(supply.nval_num) sup, max(amount.nval_num) amt +from i2b2fact m inner join pcornet_med mo +on m.concept_cd = mo.c_basecode +inner join encounter enc on enc.encounterid = m.encounter_Num + + -- jgk bugfix 11/2 - we weren't filtering dispensing events + inner join (select pcori_basecode,c_fullname,encounter_num,concept_cd from i2b2fact basis + inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num + join pcornet_med basiscode + on basis.modifier_cd = basiscode.c_basecode + and basiscode.c_fullname='\PCORI_MOD\RX_BASIS\DI\') basis + on m.encounter_num = basis.encounter_num + and m.concept_cd = basis.concept_Cd + + left join supply + on m.encounter_num = supply.encounter_num + and m.concept_cd = supply.concept_Cd + + + left join amount + on m.encounter_num = amount.encounter_num + and m.concept_cd = amount.concept_Cd + +group by m.encounter_num ,m.patient_num, m.start_date, mo.pcori_ndc; +*/ + +execute immediate 'create index dispensing_idx on dispensing (PATID)'; +GATHER_TABLE_STATS('DISPENSING'); + +end PCORNetDispensing; +/ +BEGIN +PCORNetDispensing(); +END; +/ +update cdm_status +set end_time = sysdate, records = (select count(*) from dispensing) +where task = 'dispensing' +/ +select records from cdm_status where task = 'dispensing' \ No newline at end of file diff --git a/Oracle/encounter.sql b/Oracle/encounter.sql new file mode 100644 index 0000000..f322c84 --- /dev/null +++ b/Oracle/encounter.sql @@ -0,0 +1,208 @@ +/** encounter - create and populate the encounter table. +*/ +insert into cdm_status (task, start_time) select 'encounter', sysdate from dual +/ + +BEGIN +PMN_DROPSQL('DROP TABLE encounter'); +END; +/ + +BEGIN +PMN_DROPSQL('DROP TABLE drg'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table encounter_key'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table encounter_w_drg'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table encounter_w_type'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table encounter_w_pay'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table encounter_w_fin'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table encounter_w_fac_zip'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table encounter_w_fac_id'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table encounter_w_fac_type'); +END; +/ +create table drg as +select * from +(select patient_num,encounter_num,drg_type, drg,row_number() over (partition by patient_num, encounter_num order by drg_type desc) AS rn from +(select patient_num,encounter_num,drg_type,max(drg) drg from +(select distinct f.patient_num,encounter_num,SUBSTR(c_fullname,22,2) drg_type,SUBSTR(pcori_basecode,INSTR(pcori_basecode, ':')+1,3) drg from i2b2fact f +inner join demographic d on f.patient_num=d.patid +inner join pcornet_enc enc on enc.c_basecode = f.concept_cd +and enc.c_fullname like '\PCORI\ENCOUNTER\DRG\%') drg1 group by patient_num,encounter_num,drg_type) drg) drg +where rn=1 +/ + +create index drg_idx on drg (patient_num, encounter_num) +/ + +create table encounter_key as +select patient_num PATID +, encounter_num ENCOUNTERID +, start_date ADMIT_DATE +, end_Date DISCHARGE_DATE +, providerid PROVIDERID +, admitting_source ADMITTING_SOURCE +, discharge_disposition RAW_DISCHARGE_DISPOSITION +, discharge_status RAW_DISCHARGE_STATUS +, inout_cd +from i2b2visit v +join demographic d on v.patient_num=d.patid +/ + +create table encounter_w_drg as +select en.* +, drg DRG +, drg_type DRG_TYPE +from encounter_key en +left join drg on drg.patient_num = en.patid and drg.encounter_num = en.encounterid +/ + +create table encounter_w_type as +select en.* +, nvl(SUBSTR(pcori_basecode, INSTR(pcori_basecode, ':') + 1, 2), 'UN') ENC_TYPE +, case when SUBSTR(pcori_basecode, INSTR(pcori_basecode, ':') + 1, 2) = 'AV' then 'NI' else raw_discharge_disposition end DISCHARGE_DISPOSITION +, case when SUBSTR(pcori_basecode, INSTR(pcori_basecode, ':') + 1, 2) = 'AV' then 'NI' else raw_discharge_status end DISCHARGE_STATUS +from encounter_w_drg en +left join pcornet_enc e on e.c_dimcode like '%'''||inout_cd||'''%' and e.c_fullname like '\PCORI\ENCOUNTER\ENC_TYPE\%' +/ + +-- pay_order was added as a quick fix for the problem of multiple payers mapping to the same encounterid. +-- The partition / order by operation selects the primary payer by alphabetical order (not good). +-- Other options considered were precedence by encounter type, by financial class or by CDM code but +-- none proved clear and expedient. +-- TODO: rework payer selection to align with encounter rollup logic or determine a clear precedence from other criteria. +create table encounter_w_pay as +select en.* +, pay.instance_num +, pay.tval_char RAW_PAYER_NAME_PRIMARY +, pay.concept_cd RAW_PAYER_ID_PRIMARY +from encounter_w_type en +left join + (select encounter_num + , patient_num + , instance_num + , concept_cd + , tval_char + , row_number() over (partition by patient_num, encounter_num order by tval_char) as pay_order + from i2b2fact + where concept_cd like 'O2|PAYER_PRIMARY:%' or concept_cd like 'IDX|PAYER_PRIMARY:%' + ) pay on en.patid = pay.patient_num and en.encounterid = pay.encounter_num and pay_order = 1 +/ + +-- IDX does not provide a financial class, hence the abbreviated condition. However, the payer type code can be +-- inferred from the payer name, particularly were IDX names and O2 names coincide. This might challenge the +-- do not impute rule specified in the CDM spec, but doesn't seem unreasonable. +create table encounter_w_fin as +select en.* +, pm.code PAYER_TYPE_PRIMARY +, sf.tval_char RAW_PAYER_TYPE_PRIMARY +from encounter_w_pay en +left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num and sf.source_column = 'FINANCIAL_CLASS' +left join payer_map pm on pm.payer_name = en.raw_payer_name_primary and (en.raw_payer_id_primary like 'O2|PAYER_PRIMARY:%' and pm.financial_class = sf.tval_char) + or( pm.payer_name = en.raw_payer_name_primary and (en.raw_payer_id_primary like 'IDX|PAYER_PRIMARY:%')) +/ +create table encounter_w_fac_zip as +select en.* +, sf.tval_char facility_location +from encounter_w_fin en +left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num and sf.source_column = 'FACILITY_ZIP' +/ +create table encounter_w_fac_id as +select en.* +, sf.tval_char facilityid +, sf.tval_char raw_facility_code +from encounter_w_fac_zip en +left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num and sf.source_column = 'FACILITY_ID' +/ +create table encounter_w_fac_type as +select en.* +, sf.tval_char facility_type +, sf.tval_char raw_facility_type +from encounter_w_fac_id en +left join &&i2b2_data_schema.supplemental_fact sf on en.instance_num = sf.instance_num and sf.source_column = 'FACILITYTYPE' +/ + +create table encounter as +select cast(patid as varchar(50)) PATID +, cast(encounterid as varchar(50)) ENCOUNTERID +, en.admit_date +, to_char(en.admit_date, 'HH24:MI') ADMIT_TIME +, en.discharge_date +, to_char(en.discharge_date,'HH24:MI') DISCHARGE_TIME +, en.providerid +, en.FACILITY_LOCATION +, en.enc_type +, en.FACILITYID +, cast(en.discharge_disposition as varchar(2)) DISCHARGE_DISPOSITION +, cast(en.discharge_status as varchar(2)) DISCHARGE_STATUS +, en.drg +, en.drg_type +, cast(en.admitting_source as varchar(2)) ADMITTING_SOURCE +, cast(en.payer_type_primary as varchar(5)) PAYER_TYPE_PRIMARY +, cast(null as varchar(5)) PAYER_TYPE_SECONDARY +, en.FACILITY_TYPE +, cast(null as varchar(50)) RAW_SITEID +, cast(null as varchar(50)) RAW_ENC_TYPE +, cast(null as varchar(50)) RAW_DISCHARGE_DISPOSITION +, cast(null as varchar(50)) RAW_DISCHARGE_STATUS +, cast(null as varchar(50)) RAW_DRG_TYPE +, cast(null as varchar(50)) RAW_ADMITTING_SOURCE +, en.RAW_FACILITY_TYPE +, en.raw_facility_code +, en.raw_payer_type_primary +, en.raw_payer_name_primary +, en.raw_payer_id_primary +, cast(null as varchar(50)) RAW_PAYER_TYPE_SECONDARY +, cast(null as varchar(50)) RAW_PAYER_NAME_SECONDARY +, cast(null as varchar(50)) RAW_PAYER_ID_SECONDARY +from encounter_w_fac_type en +/ + +create unique index encounter_pk on encounter (ENCOUNTERID) +/ + +create index encounter_idx on encounter (PATID, ENCOUNTERID) +/ + +BEGIN +GATHER_TABLE_STATS('ENCOUNTER'); +END; +/ + +update cdm_status +set end_time = sysdate, records = (select count(*) from encounter) +where task = 'encounter' +/ + +select records from cdm_status where task = 'encounter' diff --git a/Oracle/enrollment.sql b/Oracle/enrollment.sql new file mode 100644 index 0000000..9569479 --- /dev/null +++ b/Oracle/enrollment.sql @@ -0,0 +1,59 @@ +/** enrollment - create and populate the enrollment table. +*/ +insert into cdm_status (task, start_time) select 'enrollment', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE enrollment'); +END; +/ +CREATE TABLE enrollment ( + PATID varchar(50) NOT NULL, + ENR_START_DATE date NOT NULL, + ENR_END_DATE date NULL, + CHART varchar(1) NULL, + ENR_BASIS varchar(1) NOT NULL, + RAW_CHART varchar(50) NULL, + RAW_BASIS varchar(50) NULL +) +/ +create or replace procedure PCORNetEnroll as +begin + +PMN_DROPSQL('drop index enrollment_idx'); + +execute immediate 'truncate table enrollment'; + +INSERT INTO enrollment(PATID, ENR_START_DATE, ENR_END_DATE, CHART, ENR_BASIS) +with pats_delta as ( + -- If only one visit, visit_delta_days will be 0 + select patient_num, max(start_date) - min(start_date) visit_delta_days + from i2b2visit + where start_date > add_months(sysdate, -&&enrollment_months_back) + group by patient_num + ), +enrolled as ( + select distinct patient_num + from pats_delta + where visit_delta_days > 30 + ) +select + visit.patient_num patid, min(visit.start_date) enr_start_date, + max(visit.start_date) enr_end_date, 'Y' chart, 'A' enr_basis +from enrolled enr +join i2b2visit visit on enr.patient_num = visit.patient_num +group by visit.patient_num; + +execute immediate 'create index enrollment_idx on enrollment (PATID)'; +GATHER_TABLE_STATS('ENROLLMENT'); + +end PCORNetEnroll; +/ +BEGIN +PCORNetEnroll(); +END; +/ +update cdm_status +set end_time = sysdate, records = (select count(*) from enrollment) +where task = 'enrollment' +/ +select records from cdm_status where task = 'enrollment' \ No newline at end of file diff --git a/Oracle/etl_tests_init.sql b/Oracle/etl_tests_init.sql new file mode 100644 index 0000000..21db271 --- /dev/null +++ b/Oracle/etl_tests_init.sql @@ -0,0 +1,84 @@ +/* etl_tests_init -- Initialize ETL tests + +Copyright (c) 2015 University of Kansas Medical Center +part of the HERON* open source codebase; see NOTICE file for license details. +* http://informatics.kumc.edu/work/wiki/HERON + +References: + * KUMC Informatics ticket #2781 +*/ + +whenever sqlerror continue; +drop table etl_tests; +whenever sqlerror exit; + +create table etl_tests ( + /* Description of test */ + test_domain varchar(256) not null -- Such as top-level folder in HERON that's affected (Frontiers Registry...) + , test_name varchar(1024) not null -- User friendly name (hictr_mrn_leading_zero...) + /* Pass/Fail criteria - can define one or both of the following - INCLUSIVE */ + , lower_bound number null + , upper_bound number null + , units varchar(256) null -- What are we counting? (patients, facts, concepts, etc.) + , test_description varchar(4000) null -- More detailed description - how is the end-user affected if this test fails? + ); + +alter table etl_tests add constraint pk_etl_tests primary key (test_domain, test_name); + +-- Table to store test values (inserted during ETL) +whenever sqlerror continue; +drop table etl_test_values; +whenever sqlerror exit; + +create table etl_test_values as + select test_domain, test_name from etl_tests where 1=0; + +alter table etl_test_values + add (test_value number not null + , detail_char_1 varchar(256) + , detail_char_2 varchar(256) + , detail_num_1 number + , detail_num_2 number + , test_note varchar(256) + , result_id number not null + , result_date date not null); + +alter table etl_test_values add constraint pk_etl_test_results primary key (result_id); + +-- I wish we had "auto increment". We can use a trigger but then we need PL/SQL. +whenever sqlerror continue; +drop sequence sq_result_id; +whenever sqlerror exit; +create sequence sq_result_id; + +/* Insert some example tests for debugging. */ +-- Example test that doesn't have something in the CSV +insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) +with test_key as ( + select 'test' test_domain, + 'test_not_found_in_csv' test_name from dual + ) +select 666 test_value, test_key.*, sq_result_id.nextval, sysdate from test_key; + +-- Example successful test +insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) +with test_key as ( + select 'test' test_domain, + 'test_success' test_name from dual + ) +select 1 test_value, test_key.*, sq_result_id.nextval, sysdate from test_key; + +-- Example test with no bounds specified +insert into etl_test_values (test_value, test_domain, test_name, result_id, result_date) +with test_key as ( + select 'test' test_domain, + 'test_no_bound' test_name from dual + ) +select 1 test_value, test_key.*, sq_result_id.nextval, sysdate from test_key; + + +/* complete check */ +select 1 complete +from (select test_name from etl_test_values where 1=0 + union all select * from dual -- guarantee 1 row + ); diff --git a/Oracle/harvest.sql b/Oracle/harvest.sql new file mode 100644 index 0000000..5133d01 --- /dev/null +++ b/Oracle/harvest.sql @@ -0,0 +1,124 @@ +/** harvest - create and populate the harvest table. +*/ +insert into cdm_status (task, start_time) select 'harvest', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE harvest'); +END; +/ +CREATE TABLE harvest( + NETWORKID varchar(10) NOT NULL, + NETWORK_NAME varchar(20) NULL, + DATAMARTID varchar(10) NOT NULL, + DATAMART_NAME varchar(20) NULL, + DATAMART_PLATFORM varchar(2) NULL, + CDM_VERSION numeric(8, 2) NULL, + DATAMART_CLAIMS varchar(2) NULL, + DATAMART_EHR varchar(2) NULL, + BIRTH_DATE_MGMT varchar(2) NULL, + ENR_START_DATE_MGMT varchar(2) NULL, + ENR_END_DATE_MGMT varchar(2) NULL, + ADMIT_DATE_MGMT varchar(2) NULL, + DISCHARGE_DATE_MGMT varchar(2) NULL, + DX_DATE_MGMT varchar92) NULL, + PX_DATE_MGMT varchar(2) NULL, + RX_ORDER_DATE_MGMT varchar(2) NULL, + RX_START_DATE_MGMT varchar(2) NULL, + RX_END_DATE_MGMT varchar(2) NULL, + DISPENSE_DATE_MGMT varchar(2) NULL, + LAB_ORDER_DATE_MGMT varchar(2) NULL, + SPECIMEN_DATE_MGMT varchar(2) NULL, + RESULT_DATE_MGMT varchar(2) NULL, + MEASURE_DATE_MGMT varchar(2) NULL, + ONSET_DATE_MGMT varchar(2) NULL, + REPORT_DATE_MGMT varchar(2) NULL, + RESOLVE_DATE_MGMT varchar(2) NULL, + PRO_DATE_MGMT varchar(2) NULL, + DEATH_DATE_MGMT varchar(2) NULL, + MEDADMIN_START_DATE_MGMT varchar(2) NULL, + MEDADMIN_STOP_DATE_MGMT varchar(2) NULL, + OBSCLIN_DATE_MGMT varchar(2) NULL, + OBSGEN_DATE_MGMT varchar(2) NULL, + ADDRESS_PERIOD_START_MGMT varchar(2) NULL, + ADDRESS_PERIOD_END_MGMT varchar(2) NULL, + VX_RECORD_DATE_MGMT varchar(2) NULL, + VX_ADMIN_DATE_MGMT varchar(2) NULL, + VX_EXP_DATE_MGMT varchar(2) NULL, + REFRESH_HASH_TOKEN_DATE varchar(2) NULL, + --REFRESH_LDS_ADDRESS_HISTORY_DATE varchar(2) NULL, + --REFRESH_IMMUNIZATION_DATE varchar(2) NULL, + REFRESH_DEMOGRAPHIC_DATE date NULL, + REFRESH_ENROLLMENT_DATE date NULL, + REFRESH_ENCOUNTER_DATE date NULL, + REFRESH_DIAGNOSIS_DATE date NULL, + REFRESH_PROCEDURES_DATE date NULL, + REFRESH_VITAL_DATE date NULL, + REFRESH_DISPENSING_DATE date NULL, + REFRESH_LAB_RESULT_CM_DATE date NULL, + REFRESH_CONDITION_DATE date NULL, + REFRESH_PRO_CM_DATE date NULL, + REFRESH_PRESCRIBING_DATE date NULL, + REFRESH_PCORNET_TRIAL_DATE date NULL, + REFRESH_DEATH_DATE date NULL, + REFRESH_DEATH_CAUSE_DATE date NULL, + REFRESH_MED_ADMIN_DATE date NULL, + REFRESH_OBS_CLIN_DATE date NULL, + REFRESH_PROVIDER_DATE date NULL, + REFRESH_OBS_GEN_DATE date NULL +) +/ +create or replace procedure PCORNetHarvest as +begin + +execute immediate 'truncate table harvest'; + +INSERT INTO harvest(NETWORKID, NETWORK_NAME, DATAMARTID, DATAMART_NAME, DATAMART_PLATFORM, CDM_VERSION, DATAMART_CLAIMS, DATAMART_EHR, + BIRTH_DATE_MGMT, ENR_START_DATE_MGMT, ENR_END_DATE_MGMT, ADMIT_DATE_MGMT, DISCHARGE_DATE_MGMT, DX_DATE_MGMT, PX_DATE_MGMT, RX_ORDER_DATE_MGMT, + RX_START_DATE_MGMT, RX_END_DATE_MGMT, DISPENSE_DATE_MGMT, LAB_ORDER_DATE_MGMT, SPECIMEN_DATE_MGMT, RESULT_DATE_MGMT, MEASURE_DATE_MGMT, + ONSET_DATE_MGMT, REPORT_DATE_MGMT, RESOLVE_DATE_MGMT, PRO_DATE_MGMT, DEATH_DATE_MGMT, MEDADMIN_START_DATE_MGMT, MEDADMIN_STOP_DATE_MGMT, + OBSCLIN_DATE_MGMT, OBSGEN_DATE_MGMT, ADDRESS_PERIOD_START_MGMT, ADDRESS_PERIOD_END_MGMT,VX_RECORD_DATE_MGMT, + VX_ADMIN_DATE_MGMT,VX_EXP_DATE_MGMT,REFRESH_HASH_TOKEN_DATE, REFRESH_DEMOGRAPHIC_DATE, REFRESH_ENROLLMENT_DATE, + REFRESH_ENCOUNTER_DATE, REFRESH_DIAGNOSIS_DATE, REFRESH_PROCEDURES_DATE, REFRESH_VITAL_DATE, REFRESH_DISPENSING_DATE, + REFRESH_LAB_RESULT_CM_DATE, REFRESH_CONDITION_DATE, REFRESH_PRO_CM_DATE, REFRESH_PRESCRIBING_DATE, REFRESH_PCORNET_TRIAL_DATE, + REFRESH_DEATH_DATE, REFRESH_DEATH_CAUSE_DATE, REFRESH_MED_ADMIN_DATE, REFRESH_OBS_CLIN_DATE, REFRESH_PROVIDER_DATE, REFRESH_OBS_GEN_DATE, + REFRESH_HASH_TOKEN_DATE +) + select '&&network_id', '&&network_name', getDataMartID(), getDataMartName(), getDataMartPlatform(), 4.1, hl.DATAMART_CLAIMS, hl.DATAMART_EHR, + hl.BIRTH_DATE_MGMT, hl.ENR_START_DATE_MGMT, hl.ENR_END_DATE_MGMT, hl.ADMIT_DATE_MGMT, hl.DISCHARGE_DATE_MGMT, hl.DX_DATE_MGMT, hl.PX_DATE_MGMT, + hl.RX_ORDER_DATE_MGMT, hl.RX_START_DATE_MGMT, hl.RX_END_DATE_MGMT, hl.DISPENSE_DATE_MGMT, hl.LAB_ORDER_DATE_MGMT, hl.SPECIMEN_DATE_MGMT, + hl.RESULT_DATE_MGMT, hl.MEASURE_DATE_MGMT, hl.ONSET_DATE_MGMT, hl.REPORT_DATE_MGMT, hl.RESOLVE_DATE_MGMT, hl.PRO_DATE_MGMT, + hl.DEATH_DATE_MGMT, hl.MEDADMIN_START_DATE_MGMT, hl.MEDADMIN_STOP_DATE_MGMT, hl.OBSCLIN_DATE_MGMT, hl.OBSGEN_DATE_MGMT, + hl.ADDRESS_PERIOD_START_MGMT, hl.ADDRESS_PERIOD_END_MGMT, hl.VX_RECORD_DATE_MGMT, hl.VX_ADMIN_DATE_MGMT, hl.VX_EXP_DATE_MGMT, + case when (select records from cdm_status where task = 'demographic') > 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, + case when (select records from cdm_status where task = 'enrollment') > 0 then current_date else null end REFRESH_ENROLLMENT_DATE, + case when (select records from cdm_status where task = 'encounter') > 0 then current_date else null end REFRESH_ENCOUNTER_DATE, + case when (select records from cdm_status where task = 'diagnosis') > 0 then current_date else null end REFRESH_DIAGNOSIS_DATE, + case when (select records from cdm_status where task = 'procedures') > 0 then current_date else null end REFRESH_PROCEDURES_DATE, + case when (select records from cdm_status where task = 'vital') > 0 then current_date else null end REFRESH_VITAL_DATE, + case when (select records from cdm_status where task = 'dispensing') > 0 then current_date else null end REFRESH_DISPENSING_DATE, + case when (select records from cdm_status where task = 'lab_result_cm') > 0 then current_date else null end REFRESH_LAB_RESULT_CM_DATE, + case when (select records from cdm_status where task = 'condition') > 0 then current_date else null end REFRESH_CONDITION_DATE, + case when (select records from cdm_status where task = 'pro_cm') > 0 then current_date else null end REFRESH_PRO_CM_DATE, + case when (select records from cdm_status where task = 'prescribing') > 0 then current_date else null end REFRESH_PRESCRIBING_DATE, + case when (select records from cdm_status where task = 'pcornet_trial') > 0 then current_date else null end REFRESH_PCORNET_TRIAL_DATE, + case when (select records from cdm_status where task = 'death') > 0 then current_date else null end REFRESH_DEATH_DATE, + case when (select records from cdm_status where task = 'death_cause') > 0 then current_date else null end REFRESH_DEATH_CAUSE_DATE, + case when (select records from cdm_status where task = 'med_admin') > 0 then current_date else null end REFRESH_MED_ADMIN_DATE, + case when (select records from cdm_status where task = 'obs_clin') > 0 then current_date else null end REFRESH_OBS_CLIN_DATE, + case when (select records from cdm_status where task = 'provider') > 0 then current_date else null end REFRESH_PROVIDER_DATE, + case when (select records from cdm_status where task = 'obs_gen') > 0 then current_date else null end REFRESH_OBS_GEN_DATE, + case when (select records from cdm_status where task = 'hash_token') > 0 then current_date else null end REFRESH_HASH_TOKEN + + from harvest_local hl; + +end PCORNetHarvest; +/ +BEGIN +PCORNetHarvest(); +END; +/ +update cdm_status +set end_time = sysdate, records = (select count(*) from harvest) +where task = 'harvest' +/ +select records from cdm_status where task = 'harvest' diff --git a/Oracle/hash_token.sql b/Oracle/hash_token.sql new file mode 100644 index 0000000..11c391a --- /dev/null +++ b/Oracle/hash_token.sql @@ -0,0 +1,30 @@ +/** harvest - create and populate the hash_token table. +*/ +insert into cdm_status (task, start_time) select 'hash_token', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE hash_token'); +END; +/ + +create table HASH_TOKEN ( +PATID varchar(50) NOT NULL, +TOKEN_01 varchar(50), +TOKEN_02 varchar(50), +TOKEN_05 varchar(50), +TOKEN_12 varchar(50), +TOKEN_17 varchar(50), +TOKEN_21 varchar(50), +TOKEN_22 varchar(50), +TOKEN_23 varchar(50), +CONSTRAINT pk_hash_token_pat PRIMARY KEY (patid), +CONSTRAINT fk_hash_dem_patid + FOREIGN KEY (patid) + REFERENCES demographic (patid) +); + +/ +update cdm_status +set end_time = sysdate, records = (select count(*) from hash_token) +where task = 'hash_token' +/ \ No newline at end of file diff --git a/Oracle/heron_encounter_style.sql b/Oracle/heron_encounter_style.sql new file mode 100644 index 0000000..2db90d6 --- /dev/null +++ b/Oracle/heron_encounter_style.sql @@ -0,0 +1,410 @@ +/** heron_encounter_style -- apply HERON / KUMC / GPC encounter conventions + +copyright (c) 2016 University of Kansas Medical Center +availble under the i2b2 Software License (aka MPL?) + +*/ + +/** Index patid, encounterid in hopes of speeding up exploration. + +alter session set current_schema = pcornet_cdm; + +create unique index demographic_patid on demographic (patid); +create unique index encounter_id on encounter (encounterid); + +create index encounter_patid on encounter (patid); +create index diagnosis_patid on diagnosis (patid); +create index diagnosis_encid on diagnosis (encounterid); +create index procedures_patid on procedures (patid); +create index procedures_encid on procedures (encounterid); +create index vital_patid on vital (patid); + +actually, foreign key constraints make more sense... + +alter table encounter +add constraint fk_patid foreign key (patid) +references demographic (patid); + + +alter session set NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI'; + + */ + + +/** get length_of_stay, visit end_date from fact table + +TODO: push this back into HERON ETL +*/ + +--TODO: Consider trying to share this function with PCORNetLoad_ora.sql +create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS + BEGIN + DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. + tabname => table_name, + estimate_percent => 50, -- Percentage picked somewhat arbitrarily + cascade => TRUE, + degree => 16 + ); +END GATHER_TABLE_STATS; +/ + + +merge into "&&i2b2_data_schema".visit_dimension vd +using ( + -- can we get it from UHC length_of_stay? + select obs.encounter_num, obs.patient_num + , obs.nval_num length_of_stay + , obs.end_date discharge_date + from "&&i2b2_data_schema".observation_fact obs + where obs.concept_cd = 'UHC|LOS:1' -- TODO: parameterize +) obs +on ( obs.encounter_num = vd.encounter_num + and obs.patient_num = vd.patient_num) +when matched then update + set vd.length_of_stay = obs.length_of_stay + , vd.end_date = obs.discharge_date +; +-- 194,328 rows merged. + +merge into "&&i2b2_data_schema".visit_dimension vd +using ( + -- how about discharge disposition? + select obs.encounter_num, obs.patient_num + , max(obs.start_date) discharge_date + from "&&i2b2_data_schema".observation_fact obs + join ( + select distinct concept_cd + from "&&i2b2_data_schema".concept_dimension + -- TODO: map to PCORnet path + where concept_path like '\i2b2\Visit Details\Discharge Disposition Codes\%' + group by concept_cd + ) cd on cd.concept_cd = obs.concept_cd + group by encounter_num, patient_num +) obs +on ( obs.encounter_num = vd.encounter_num + and obs.patient_num = vd.patient_num) +when matched then update + set vd.length_of_stay = obs.discharge_date - vd.start_date + , vd.end_date = obs.discharge_date +; +-- 3,879,931 rows merged. + + +/* TODO: get encounter type from source system? */ +/* TODO: Consider moving to cdm_transform_tests.sql +select count(*), sourcesystem_cd +from nightherondata.visit_dimension +group by sourcesystem_cd; +*/ + +/* TODO: get encounter type from place of service? */ +/* TODO: Consider moving to cdm_transform_tests.sql +select obs.encounter_num, obs.patient_num, obs.start_date, obs.end_date + , con.concept_cd, con.name_char place_of_service + , obs.sourcesystem_cd +from "&&i2b2_data_schema".observation_fact obs +join "&&i2b2_data_schema".concept_dimension con on con.concept_cd = obs.concept_cd +join "&&i2b2_data_schema".visit_dimension v on v.encounter_num = obs.encounter_num +where con.concept_path like '\i2b2\Visit Details\Place of Service (IDX)\%' +and v.inout_cd = 'OT' +; +*/ + +/* Update the visit dimension to populate the inout_cd with the appropriate +PCORNet encounter type codes based on data in the observation_fact. + +The pcornet_mapping table contains columns that translate the PCORNet paths to +local paths. + +Example: +pcori_path,local_path +\PCORI\ENCOUNTER\ENC_TYPE\AV\,\i2b2\Visit Details\ENC_TYPE\AV\ +*/ +select * from pcornet_mapping pm where 1=0; + +whenever sqlerror continue; +drop table enc_type; +whenever sqlerror exit; + +/* explore encounter mapping: where do our encounters come from? */ +/* +select count(*), count(distinct encounter_num), encounter_ide_source +from "&&i2b2_data_schema".encounter_mapping +group by encounter_ide_source +order by 1 desc +; + +select min(encounter_num), encounter_ide_source +from "&&i2b2_data_schema".encounter_mapping +group by encounter_ide_source +order by 1 +; +*/ +whenever sqlerror continue; +drop table enc_type_merge; +whenever sqlerror exit; + +create table enc_type_merge as +with + enc_type_codes as ( + select pm.pcori_path + , replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code + , pm.local_path + , cd.concept_cd + from pcornet_mapping pm + join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' + where pm.pcori_path like '\PCORI\ENCOUNTER\ENC_TYPE\%' + ) + /* Note: our patient-day style encounters etc. may result in + multiple encounter types for a single encounter. + Therefore we need to pivot to all the encounter types associated with each + encounter and then choose (intentionally and deterministically) the + single encounter type that will prevail for each encounter + */ + , obs_enc_type_pivot as ( + select * + from (select obs.encounter_num, et.pcori_code + from "&&i2b2_data_schema".observation_fact obs + join enc_type_codes et on et.concept_cd = obs.concept_cd) + pivot + ( + max(pcori_code) + for pcori_code in ('EI' as ei, 'IP' as ip, 'ED' as ed, 'IS' as e_is + , 'AV' as av, 'OA' as oa, 'OT' as ot, 'OS' as os + , 'NI' as ni, 'UN' as un) + ) + ) + /* Single Encounter type to Encounter coercion: + IP + ED = EI + EI > IP || ED > OS > IS > AV > OA > OT > NI > UN + */ + select encounter_num, + case when ip is not null and ed is not null then 'EI' + else coalesce(ei, ip, ed, os, e_is, av, oa, ot, ni, un, 'UN') + end as pcori_code + from obs_enc_type_pivot; + +--select pcori_code, count(*) from enc_type_merge group by pcori_code order by pcori_code; + +merge into "&&i2b2_data_schema".visit_dimension vd +using enc_type_merge et +on ( vd.encounter_num = et.encounter_num ) +when matched then update +set inout_cd = et.pcori_code; +-- 11,085,911 rows merged. + +/* Discharge disposition +Valid values are: + +A=Discharged alive +E=Expired +NI=No information +UN=Unknown +OT=Other + +So, if there are multiple discharge dispositions per encounter (as we've observed +in test data at least) prioritize and pick one. +*/ +whenever sqlerror continue; +alter table "&&i2b2_data_schema".visit_dimension add ( + discharge_disposition VARCHAR2(4 BYTE) + ); +drop table enc_disp_facts; +drop table discharge_disp; +whenever sqlerror exit; + +-- Two steps (tables) due to Oracle performance/picking bad plans when it's all one query +create table enc_disp_facts as +with +enc_type_codes as ( + select + pm.pcori_path, replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code, pm.local_path, cd.concept_cd + from pcornet_mapping pm + join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' + where pm.pcori_path like '\PCORI\ENCOUNTER\DISCHARGE_DISPOSITION\%' + ) +select obs.encounter_num, et.pcori_code +from "&&i2b2_data_schema".observation_fact obs +join enc_type_codes et on et.concept_cd = obs.concept_cd; + +create table discharge_disp as +with +ranks as ( + select + encounter_num, pcori_code, + -- Prioritize disposition + decode(pcori_code, 'E', 0, 'A', 1, 'OT', 2, 'UN', 3, 'NI', 4, 99) pc_rank + from enc_disp_facts + ), +priority_rank as ( + -- TODO: Consider figuring out why we have multiple discharge dispositions per encounter. + select min(pc_rank) pc_rank, encounter_num + from ranks + group by encounter_num + ) +select distinct + ranks.encounter_num, coalesce(ranks.pcori_code, 'NI') pcori_code +from priority_rank pr +join ranks on pr.encounter_num = ranks.encounter_num and pr.pc_rank = ranks.pc_rank +; + +create index discharge_disp_encnum_idx on discharge_disp(encounter_num); +begin +GATHER_TABLE_STATS('discharge_disp'); +end; +/ + +update "&&i2b2_data_schema".visit_dimension vd +set discharge_disposition = coalesce(( + select dd.pcori_code from discharge_disp dd + where vd.encounter_num = dd.encounter_num + ), 'UN'); + + + /* Discharge status +There may be multiple per encounter - rank and pick one. +*/ +whenever sqlerror continue; +alter table "&&i2b2_data_schema".visit_dimension add ( + discharge_status VARCHAR2(4 BYTE) + ); +drop table enc_status_facts; +drop table discharge_status; +whenever sqlerror exit; + +-- Two steps (tables) due to Oracle performance/picking bad plans when it's all one query +create table enc_status_facts as +with +enc_status_codes as ( + select + pm.pcori_path, replace(substr(pm.pcori_path, instr(pm.pcori_path, '\', -2)), '\', '') pcori_code, pm.local_path, cd.concept_cd + from pcornet_mapping pm + join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' + where pm.pcori_path like '\PCORI\ENCOUNTER\DISCHARGE_STATUS\%' + ) +select obs.encounter_num, et.pcori_code +from "&&i2b2_data_schema".observation_fact obs +join enc_status_codes et on et.concept_cd = obs.concept_cd; + +create index enc_status_facts_encnum_idx on enc_status_facts(encounter_num); +begin +GATHER_TABLE_STATS('enc_status_facts'); +end; +/ + +create table discharge_status as +with +ranks as ( + select + encounter_num, pcori_code, + -- Prioritize status: First EX, Middle: , Last OT, UN, NI + decode(pcori_code, 'EX',0,'AF',1,'AL',2,'AM',3,'AW',4,'HH',5,'HO',6,'HS',7, + 'IP',8,'NH',9,'RH',10,'RS',11,'SH',12,'SN',12,'OT',96, + 'UN',97,'NI',98,99) pc_rank + from enc_status_facts + ), +priority_rank as ( + -- TODO: Consider figuring out why we have multiple discharge status values per encounter. + select min(pc_rank) pc_rank, encounter_num + from ranks + group by encounter_num + ) +select distinct + ranks.encounter_num, coalesce(ranks.pcori_code, 'NI') pcori_code +from priority_rank pr +join ranks on pr.encounter_num = ranks.encounter_num and pr.pc_rank = ranks.pc_rank +; + +create index discharge_status_encnum_idx on discharge_status(encounter_num); +begin +GATHER_TABLE_STATS('discharge_status'); +end; +/ + +update "&&i2b2_data_schema".visit_dimension vd +set discharge_status = coalesce(( + select ds.pcori_code from discharge_status ds + where vd.encounter_num = ds.encounter_num + ), 'UN'); + + +/* Admitting source +There may be multiple per encounter - rank and pick one. +*/ +whenever sqlerror continue; +alter table "&&i2b2_data_schema".visit_dimension add ( + admitting_source VARCHAR2(4 BYTE) + ); +drop table admit_source_enc_code_rank; +drop table admit_source_priority_rank; +whenever sqlerror exit; + +create table admit_source_enc_code_rank as +with codes as ( + select + pm.pcori_path, cd.concept_cd, cd.name_char, + replace(pe.pcori_basecode, 'ADMITTING_SOURCE:', '') pcori_code + from + pcornet_mapping pm + join "&&i2b2_data_schema".concept_dimension cd on cd.concept_path like pm.local_path || '%' + join "&&i2b2_meta_schema".pcornet_enc pe on pe.c_fullname = pm.pcori_path + where pm.pcori_path like '\PCORI\ENCOUNTER\ADMITTING_SOURCE\%' + ) +select + obs.encounter_num, codes.pcori_code, + -- Prioritize status: Last OT, UN, NI + decode(codes.pcori_code, 'AF',0,'AL',1,'AV',2,'ED',3,'HH',4,'HO',5,'HS',6,'IP',7, + 'NH',8,'RH',9,'RS',10,'SN',11,'OT',12,'UN',13,'NI',14, 99) as_rank +from "&&i2b2_data_schema".observation_fact obs +join codes on codes.concept_cd = obs.concept_cd; + +create index admit_source_enc_rank_idx on admit_source_enc_code_rank(encounter_num, as_rank); +begin +GATHER_TABLE_STATS('admit_source_enc_code_rank'); +end; +/ + +create table admit_source_priority_rank as +select min(as_rank) as_rank, encounter_num +from admit_source_enc_code_rank ranks +group by encounter_num; + +create index admit_source_pr_enc_rank_idx on admit_source_priority_rank(encounter_num, as_rank); +begin +GATHER_TABLE_STATS('admit_source_priority_rank'); +end; +/ + +merge into "&&i2b2_data_schema".visit_dimension vd using ( + select distinct + ranks.encounter_num, coalesce(ranks.pcori_code, 'NI') pcori_code + from admit_source_priority_rank pr + join admit_source_enc_code_rank ranks on pr.encounter_num = ranks.encounter_num + and pr.as_rank = ranks.as_rank + ) admt_enc on (admt_enc.encounter_num = vd.encounter_num) +when matched then update +set vd.admitting_source = admt_enc.pcori_code +; + + +/* -- Manually update encounter (i2p-transform should fill in the column) + +select count(*) from encounter; +-- 14,665,341 + +merge into encounter pe +using "&&i2b2_data_schema".visit_dimension vd + on ( pe.encounterid = vd.encounter_num + and vd.end_date is not null) +when matched then update set pe.discharge_date = vd.end_date; +-- 165,292 rows merged. + +merge into encounter pe +using "&&i2b2_data_schema".visit_dimension vd + on ( pe.encounterid = vd.encounter_num + and vd.inout_cd is not null) +when matched then update + set pe.enc_type = vd.inout_cd; + -- 14,665,341 rows merged. +*/ diff --git a/Oracle/lab_result_cm.sql b/Oracle/lab_result_cm.sql new file mode 100644 index 0000000..6535e0c --- /dev/null +++ b/Oracle/lab_result_cm.sql @@ -0,0 +1,201 @@ +/** lab_result_cm - create and populate the lab_result_cm table. +*/ +insert into cdm_status (task, start_time) select 'lab_result_cm', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE lab_result_cm'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table lab_result_key'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table lab_result_w_base'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table lab_result_w_parent'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table lab_result_w_norm'); +END; +/ + +BEGIN +PMN_DROPSQL('drop table lab_result_w_source'); +END; +/ + +BEGIN +PMN_DROPSQL('DROP SEQUENCE lab_result_cm_seq'); +END; +/ + +create sequence lab_result_cm_seq cache 2000 +/ + +/* +concept_cd between 'KUH|COMPONENT_ID:' and 'KUH|COMPONENT_ID:~' +works in place of c_fullname like '\PCORI\LAB_RESULT_CM\%' +because there are no leaves ('L_') outside this range. +See select below for example. + +select c_name, c_basecode +from blueheronmetadata.heron_terms +where c_fullname like '\i2b2\Lab%' +and c_basecode not between 'KUH|COMPONENT_ID:' and 'KUH|COMPONENT_ID:~' +and c_visualattributes like 'L_' +; +*/ +create table lab_result_key as +select lab_result_cm_seq.nextval LAB_RESULT_CM_ID +, patient_num PATID +, encounter_num ENCOUNTERID +, instance_num INSTANCE_NUM +, provider_id PROVIDERID +, start_date LAB_ORDER_DATE +, end_date RESULT_DATE +, nval_num RESULT_NUM +, tval_char RESULT_MODIFIER +, units_cd RESULT_UNIT +, valueflag_cd ABN_IND +, valtype_cd RAW_RESULT +, concept_cd RAW_FACILITY_CODE +from &&i2b2_data_schema.observation_fact m +join encounter enc on enc.patid = m.patient_num and enc.encounterid = m.encounter_Num +where concept_cd between 'KUH|COMPONENT_ID:' and 'KUH|COMPONENT_ID:~' +and modifier_cd in ('@') -- exclude analyitics: Labs|Aggregate:Median, ... +and m.valtype_cd in ('N','T') +and (m.nval_num is null or m.nval_num<=9999999) +/ + +create table lab_result_w_base as +select lab.* +, pl.pcori_basecode LAB_LOINC +, pl.c_path C_PATH +from lab_result_key lab +left join +( + select c_basecode, c_path, pcori_basecode + from pcornet_lab + where c_fullname like '\PCORI\LAB_RESULT_CM\%' +) pl +on lab.RAW_FACILITY_CODE = pl.c_basecode +/ + +create table lab_result_w_parent as +select lab.* +, parent.c_basecode LAB_NAME +from lab_result_w_base lab +left join +( + select c_fullname, c_basecode + from pcornet_lab +) parent +on lab.C_PATH = parent.c_fullname +/ + +create table lab_result_w_norm as +select lab.* +, ref_lo NORM_RANGE_LOW +, ref_hi NORM_RANGE_HIGH +from lab_result_w_parent lab +left join +( + select patid, concept_cd, birth_date, age_lower, age_upper, ref_hi, ref_lo + from labnormal + join demographic + on demographic.sex = labnormal.sex +) norm +on lab.PATID = norm.patid +and lab.RAW_FACILITY_CODE = norm.concept_cd +and (lab.LAB_ORDER_DATE - norm.birth_date) > norm.age_lower +and (lab.LAB_ORDER_DATE - norm.birth_date) <= norm.age_upper +/ + +create table lab_result_w_source as +select lab.* +, NVL(code, 'NI') SPECIMEN_SOURCE +from lab_result_w_norm lab +left join +( + select distinct instance_num, NVL(code, 'OT') code + from &&i2b2_data_schema.supplemental_fact sf + left join specimen_source_map ssm on lower(sf.tval_char) = lower(ssm.specimen_source_name) + and ssm.specimen_source_name is not null +) map +on lab.instance_num = map.instance_num +/ + +create table lab_result_cm as +select distinct cast(lab.LAB_RESULT_CM_ID as varchar(19)) LAB_RESULT_CM_ID +, cast(lab.PATID as varchar(50)) PATID +, cast(lab.ENCOUNTERID as varchar(50)) ENCOUNTERID +, lab.SPECIMEN_SOURCE +, cast(nvl(lab.LAB_LOINC, 'NI') as varchar(10)) LAB_LOINC +, 'NI' PRIORITY +, 'NI' RESULT_LOC +, cast(nvl(lab.LAB_LOINC, 'NI') as varchar(11)) LAB_PX +, 'LC' LAB_PX_TYPE +, lab.LAB_ORDER_DATE LAB_ORDER_DATE +, lab.LAB_ORDER_DATE SPECIMEN_DATE +, to_char(lab.LAB_ORDER_DATE, 'HH24:MI') SPECIMEN_TIME +, lab.RESULT_DATE +, to_char(lab.RESULT_DATE, 'HH24:MI') RESULT_TIME +, 'NI' RESULT_QUAL +, cast(null as varchar(50)) RESULT_SNOMED +, case when lab.RAW_RESULT = 'N' then lab.RESULT_NUM else null end RESULT_NUM +, case when lab.RAW_RESULT = 'N' then (case nvl(nullif(lab.RESULT_MODIFIER, ''),'NI') when 'E' then 'EQ' when 'NE' then 'OT' when 'L' then 'LT' when 'LE' then 'LE' when 'G' then 'GT' when 'GE' then 'GE' else 'NI' end) else 'TX' end RESULT_MODIFIER +, case + when instr(lab.RESULT_UNIT, '%') > 0 then 'PERCENT' + when lab.RESULT_UNIT is null then nvl(lab.RESULT_UNIT, 'NI') + when length(lab.RESULT_UNIT) > 11 then substr(lab.RESULT_UNIT, 1, 11) + else trim(replace(upper(lab.RESULT_UNIT), '(CALC)', '')) + end RESULT_UNIT +, cast(lab.NORM_RANGE_LOW as varchar(10)) NORM_RANGE_LOW +, case + when lab.NORM_RANGE_LOW is not null and lab.NORM_RANGE_HIGH is not null then 'EQ' + when lab.NORM_RANGE_LOW is not null and lab.NORM_RANGE_HIGH is null then 'GE' + when lab.NORM_RANGE_LOW is null and lab.NORM_RANGE_HIGH is not null then 'NO' + else 'NI' + end NORM_MODIFIER_LOW +, cast(lab.NORM_RANGE_HIGH as varchar(10)) NORM_RANGE_HIGH +, case + when lab.NORM_RANGE_LOW is not null and lab.NORM_RANGE_HIGH is not null then 'EQ' + when lab.NORM_RANGE_LOW is not null and lab.NORM_RANGE_HIGH is null then 'NO' + when lab.NORM_RANGE_LOW is null and lab.NORM_RANGE_HIGH is not null then 'LE' + else 'NI' + end NORM_MODIFIER_HIGH +, case nvl(nullif(lab.ABN_IND, ''), 'NI') when 'H' then 'AH' when 'L' then 'AL' when 'A' then 'AB' else 'NI' end ABN_IND +, cast(null as varchar(50)) RAW_LAB_NAME +, cast(null as varchar(50)) RAW_LAB_CODE +, cast(null as varchar(50)) RAW_PANEL +, case when lab.RAW_RESULT = 'T' then substr(lab.RESULT_MODIFIER, 1, 50) else to_char(lab.RESULT_NUM) end RAW_RESULT +, cast(null as varchar(50)) RAW_UNIT +, cast(null as varchar(50)) RAW_ORDER_DEPT +, lab.RAW_FACILITY_CODE RAW_FACILITY_CODE +, 'PC' as lab_loinc_source +, 'OD' as lab_result_source +from lab_result_w_source lab +/ + +create index lab_result_cm_idx on lab_result_cm (PATID, ENCOUNTERID) +/ + +BEGIN +GATHER_TABLE_STATS('LAB_RESULT_CM'); +END; +/ + +update cdm_status +set end_time = sysdate, records = (select count(*) from lab_result_cm) +where task = 'lab_result_cm' +/ + +select records from cdm_status where task = 'lab_result_cm' diff --git a/Oracle/load_csv.py b/Oracle/load_csv.py new file mode 100644 index 0000000..c2fef6b --- /dev/null +++ b/Oracle/load_csv.py @@ -0,0 +1,172 @@ +''' load_csv.py - Load a specified data file with sqlplus/sqlldr +''' +from collections import defaultdict +from contextlib import contextmanager +from csv import DictReader +from subprocess import PIPE +import logging + +# OCAP exception for logging +logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', + datefmt='%Y.%m.%d %H:%M:%S', level=logging.INFO) +log = logging.getLogger(__name__) + + +def main(argv, environ, open_argv, mk_tool): + # TODO: Consider docopt + table_name, csv, ctl_out_fn, user_env, password_env = argv[1:6] + + table_tool = mk_tool(table_name=table_name, + user=environ[user_env], + password=environ[password_env]) + + with open_argv(csv, 'rb') as fin: + dr = DictReader(fin) + table_tool.create(dr=dr) + + with open_argv(ctl_out_fn, 'wb') as ctl_fout: + ctl_code = table_tool.ctl_from_csv(fields=dr.fieldnames) + ctl_fout.write(ctl_code) + + table_tool.load(ctl_fn=ctl_out_fn, csv=csv) + + +class MockPopen(object): + def __init__(self, cmd_args, stdout=None, stderr=None, shell=False): + self._cmd_args = cmd_args + self.returncode = 0 + + def communicate(self): + log.info('MockPopen::communicate: %s' % (self._cmd_args)) + return ('', '') + + +class TableTool(object): + ''' + Test DDL generation - note test for 0-length column + >>> from StringIO import StringIO + >>> sio = StringIO('\\n'.join([l.strip() for l in """col1,col2,col3 + ... some data,Some other data and stuff,""".split('\\n')])) + >>> s = TableTool('me', 'sekret', 'some_table', MockPopen) + >>> print s.ddl_from_csv(DictReader(sio)) + ... # doctest: +NORMALIZE_WHITESPACE + create table some_table ( + col1 varchar2(16), + col2 varchar2(32), + col3 varchar2(16) + ); + + Test control file generation + >>> print s.ctl_from_csv(['col1', 'col2']) + ... # doctest: +NORMALIZE_WHITESPACE + options (errors=0, skip=1) + load data + truncate into table some_table + fields terminated by ',' optionally enclosed by '"' + trailing nullcols( + col1, + col2 + ) + ''' + def __init__(self, Popen, user, password, table_name, + sqlplus_cmd='sqlplus /nolog', + sqlldr_cmd='sqlldr'): + def run(template): + cmd = dedent(template.format( + sqlplus=sqlplus_cmd, sqlldr=sqlldr_cmd, + user=user, password=password, + table_name=table_name)) + p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) + stdout, stderr = p.communicate() + if stdout: + log.info('stdout: %s', stdout) + if stderr: + log.warn('stderr: %s', stderr) + if p.returncode: + raise IOError(p.returncode) + + self.run = run + self.table_name = table_name + + @classmethod + def make(cls, Popen, user, password, name): + return TableTool(Popen, user, password, name) + + def create(self, dr): + # Note: shell = true might not be the most secure, but it makes + # piplining easier. + # See also https://docs.python.org/2.7/library/subprocess.html#replacing-shell-pipeline # noqa + # Also, be warned that SQL injection is possible with arguments! + cmd = ("""{sqlplus} < 0), + constraint chunk_num_in_range check (chunk_num between 1 and chunk_qty), + constraint chunk_first check (patient_num_first is not null or chunk_num = 1) + ) +/ +insert into patient_chunks ( + chunk_qty + , chunk_num + , patient_num_qty + , patient_num_first + , patient_num_last +) +select :chunk_qty + , chunk_num + , count(distinct patient_num) patient_num_qty + , min(patient_num) patient_num_first + , max(patient_num) patient_num_last + from ( + select patient_num, ntile(:chunk_qty) over (order by patient_num) as chunk_num + from (select distinct patient_num from BLUEHERONDATA.patient_dimension)) +group by chunk_num, :chunk_qty +order by chunk_num +/ +select case + when (select count(distinct chunk_num) + from patient_chunks + where chunk_qty = &&chunk_qty) = &&chunk_qty + then 1 + else 0 + end complete +from dual \ No newline at end of file diff --git a/Oracle/pcornet_init.sql b/Oracle/pcornet_init.sql new file mode 100644 index 0000000..13f2aab --- /dev/null +++ b/Oracle/pcornet_init.sql @@ -0,0 +1,254 @@ +/** pcornet_init - create helper functions and procedures +*/ +-- Make sure the ethnicity code has been added to the patient dimension +-- See update_ethnicity_pdim.sql +select ethnicity_cd from "&&i2b2_data_schema".patient_dimension where 1=0 +/ +-- Make sure that the providerid column has been added to the visit dimension +select providerid from "&&i2b2_data_schema".visit_dimension where 1=0 +/ +-- Make sure the inout_cd has been populated +-- See heron_encounter_style.sql +select case when qty = 0 then 1/0 else 1 end inout_cd_populated from ( + select count(*) qty from "&&i2b2_data_schema".visit_dimension where inout_cd is not null + ) +/ +-- Make sure the RXNorm mapping table exists +select rxcui from "&&i2b2_etl_schema".clarity_med_id_to_rxcui@id where 1=0 +/ + +BEGIN +dbms_stats.gather_table_stats( OWNNAME => '"&&i2b2_data_schema"', TABNAME => '"SUPPLEMENTAL_FACT"', +ESTIMATE_PERCENT => DBMS_STATS.AUTO_SAMPLE_SIZE, DEGREE => 16, CASCADE => TRUE ); +END; +/ + +create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS + BEGIN + DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. + tabname => table_name, + estimate_percent => 50, -- Percentage picked somewhat arbitrarily + cascade => TRUE, + degree => 16 + ); +END GATHER_TABLE_STATS; +/ + + +create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS + BEGIN + EXECUTE IMMEDIATE sqlstring; + EXCEPTION + WHEN OTHERS THEN NULL; +END PMN_DROPSQL; +/ + + +create or replace FUNCTION PMN_IFEXISTS(objnamestr VARCHAR2, objtypestr VARCHAR2) RETURN BOOLEAN AS +cnt NUMBER; +BEGIN + SELECT COUNT(*) + INTO cnt + FROM USER_OBJECTS + WHERE upper(OBJECT_NAME) = upper(objnamestr) + and upper(object_type) = upper(objtypestr); + + IF( cnt = 0 ) + THEN + --dbms_output.put_line('NO!'); + return FALSE; + ELSE + --dbms_output.put_line('YES!'); + return TRUE; + END IF; + +END PMN_IFEXISTS; +/ + + +create or replace PROCEDURE PMN_Execuatesql(sqlstring VARCHAR2) AS +BEGIN + EXECUTE IMMEDIATE sqlstring; + dbms_output.put_line(sqlstring); +END PMN_ExecuateSQL; +/ + + +--ACK: http://dba.stackexchange.com/questions/9441/how-to-catch-and-handle-only-specific-oracle-exceptions +create or replace procedure create_error_table(table_name varchar2) as +sqltext varchar2(4000); + +begin + dbms_errlog.create_error_log(dml_table_name => table_name); +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -955 THEN + NULL; -- suppresses ORA-00955 exception ("name is already used by an existing object") + ELSE + RAISE; + END IF; +-- Delete rows from a previous run in case the table already existed +sqltext := 'delete from ERR$_' || table_name; +PMN_Execuatesql(sqltext); +end; +/ + + +create or replace FUNCTION GETDATAMARTID RETURN VARCHAR2 IS +BEGIN + RETURN '&&datamart_id'; +END; +/ + + +CREATE OR REPLACE FUNCTION GETDATAMARTNAME RETURN VARCHAR2 AS +BEGIN + RETURN '&&datamart_name'; +END; +/ + + +CREATE OR REPLACE FUNCTION GETDATAMARTPLATFORM RETURN VARCHAR2 AS +BEGIN + RETURN '02'; -- 01 is MSSQL, 02 is Oracle +END; +/ + +BEGIN +PMN_DROPSQL('DROP TABLE cdm_status'); +END; +/ + +CREATE TABLE PCORNET_CDM.CDM_STATUS + ( + TASK VARCHAR2(50 BYTE) primary key, + START_TIME DATE NULL, + END_TIME DATE NULL, + RECORDS NUMBER(*,0) NULL + ) +/ +insert into cdm_status (task, start_time) select 'pcornet_init', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE pcornet_codelist'); +END; +/ +create table pcornet_codelist(codetype varchar2(20), code varchar2(50)) +/ + +create or replace procedure pcornet_parsecode (codetype in varchar, codestring in varchar) as + +tex varchar(2000); +pos number(9); +readstate char(1) ; +nextchar char(1) ; +val varchar(50); + +begin + +val:=''; +readstate:='F'; +pos:=0; +tex := codestring; +FOR pos IN 1..length(tex) +LOOP +-- dbms_output.put_line(val); + nextchar:=substr(tex,pos,1); + if nextchar!=',' then + if nextchar='''' then + if readstate='F' then + val:=''; + readstate:='T'; + else + insert into pcornet_codelist values (codetype,val); + val:=''; + readstate:='F' ; + end if; + else + if readstate='T' then + val:= val || nextchar; + end if; + end if; + end if; +END LOOP; + +end pcornet_parsecode; +/ + +create or replace procedure pcornet_popcodelist as + +codedata varchar(2000); +onecode varchar(20); +codetype varchar(20); + +cursor getcodesql is +select 'RACE',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' +union +select 'SEX',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' +union +select 'HISPANIC',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%'; + +begin +open getcodesql; +LOOP + fetch getcodesql into codetype,codedata; + EXIT WHEN getcodesql%NOTFOUND ; + pcornet_parsecode (codetype,codedata ); +end loop; + +close getcodesql ; +end pcornet_popcodelist; +/ + +CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT +/ + +CREATE OR REPLACE SYNONYM I2B2MEDFACT FOR OBSERVATION_FACT_MEDS +/ + +BEGIN +PMN_DROPSQL('DROP TABLE i2b2patient_list'); +END; +/ + +CREATE table i2b2patient_list as +select * from +( +select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') +) where ROWNUM<100000000 +/ + +create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) +/ + +create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE 1 +order by c_basecode +; +*/ +create table prescribing_w_cui as +select rx.* +, mo.pcori_cui rxnorm_cui +, substr(mo.c_basecode, 1, 50) raw_rxnorm_cui +, substr(mo.c_name, 1, 50) raw_rx_med_name +from prescribing_key rx +left join + (select distinct c_basecode + , c_name + , pcori_cui + from pcornet_med + ) mo +on rx.concept_cd = mo.c_basecode +/ + +/** prescribing_w_refills +This join isn't guaranteed not to introduce more rows, +but at least one measurement showed it does not. + */ +create table prescribing_w_refills as +select rx.* +, refills.nval_num rx_refills +from prescribing_w_cui rx +left join + (select instance_num + , concept_cd + , nval_num + from &&i2b2_data_schema.observation_fact + where modifier_cd = 'RX_REFILLS' + /* aka: + select c_basecode from pcornet_med refillscode + where refillscode.c_fullname like '\PCORI_MOD\RX_REFILLS\' */ + ) refills on refills.instance_num = rx.instance_num and refills.concept_cd = rx.concept_cd +/ + +create table prescribing_w_freq as +select rx.* +-- '09' as PRN has been eliminated from the list of valid frequencies and replaced with a PRN flag. +-- For backward compatibility and simplified mapping, '09' has been retained in frequency mapping +-- (see heron:med_freq_mod_map.csv) and overridden here. +, case + when substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) = '09' then 'OT' + else substr(freq.pcori_basecode, instr(freq.pcori_basecode, ':') + 1, 2) + end rx_frequency +from prescribing_w_refills rx +left join + (select instance_num + , concept_cd + , pcori_basecode + from &&i2b2_data_schema.observation_fact + join pcornet_med on modifier_cd = c_basecode + and c_fullname like '\PCORI_MOD\RX_FREQUENCY\%' + ) freq on freq.instance_num = rx.instance_num and freq.concept_cd = rx.concept_cd +/ + +create table prescribing_w_quantity as +select rx.* +, quantity.nval_num rx_quantity +from prescribing_w_freq rx +left join + (select instance_num + , concept_cd + , nval_num + from &&i2b2_data_schema.observation_fact + where modifier_cd = 'RX_QUANTITY' + /* aka: + select c_basecode from pcornet_med refillscode + where refillscode.c_fullname like '\PCORI_MOD\RX_QUANTITY\' */ + ) quantity on quantity.instance_num = rx.instance_num and quantity.concept_cd = rx.concept_cd +/ + +create table prescribing_w_supply as +select rx.* +, supply.nval_num rx_days_supply +from prescribing_w_quantity rx +left join + (select instance_num + , concept_cd + , nval_num + from &&i2b2_data_schema.observation_fact + where modifier_cd = 'RX_DAYS_SUPPLY' + /* aka: + select c_basecode from pcornet_med refillscode + where refillscode.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\' */ + ) supply on supply.instance_num = rx.instance_num and supply.concept_cd = rx.concept_cd +/ + +create table prescribing_w_basis as +select rx.* +, substr(basis.pcori_basecode, instr(basis.pcori_basecode, ':') + 1, 2) rx_basis +from prescribing_w_supply rx +left join + (select instance_num + , concept_cd + , pcori_basecode + from &&i2b2_data_schema.observation_fact + join pcornet_med on modifier_cd = c_basecode + and c_fullname like '\PCORI_MOD\RX_BASIS\%' + and modifier_cd in ('MedObs:Inpatient', 'MedObs:Outpatient') + ) basis on basis.instance_num = rx.instance_num and basis.concept_cd = rx.concept_cd +/ + +create table prescribing_w_prn as +select rx.* +-- PRN is determined by a specific source system fact or a frequency of '09'. +-- Note that '09' is now mapped to a frequency of other. See comments in +-- 'create table prescribing_w_freq as' for related logic. +, case + when prn.tval_char = 'Y' or rx.rx_frequency = '09' then 'Y' + else 'N' + end rx_prn_flag +from prescribing_w_basis rx +left join + (select instance_num + , concept_cd + , 'Y' as tval_char + from &&i2b2_data_schema.observation_fact + where modifier_cd = 'MedObs:PRN' + /* aka: + select c_basecode from pcornet_med code + where code.c_fullname like '\PCORI_MOD\RX_BASIS\PR\02\MedObs:PRN\' */ + ) prn on prn.instance_num = rx.instance_num and prn.concept_cd = rx.concept_cd +/ + +create table prescribing_w_route as +select rx.* +, nvl(rm.code, 'NI') rx_route +, rt.tval_char raw_rx_route +from prescribing_w_prn rx +left join + (select instance_num + , tval_char + from &&i2b2_data_schema.supplemental_fact + where source_column = 'PRESCRIBING_ROUTE' + ) rt on rt.instance_num = rx.instance_num +left join route_map rm on lower(rt.tval_char) = lower(rm.route_name) +/ + +create table prescribing_w_daw as +select rx.* +, cast(nvl(tval_char, 'NI') as varchar(2)) rx_dispense_as_written +from prescribing_w_route rx +left join + (select instance_num + , tval_char + from &&i2b2_data_schema.supplemental_fact + where source_column = 'DISPENSE_AS_WRITTEN' + ) daw on daw.instance_num = rx.instance_num +/ + +create table prescribing as +select rx.prescribingid +, rx.patid +, rx.encounterid +, rx.rx_providerid +, trunc(rx.start_date) rx_order_date +, to_char(rx.start_date, 'HH24:MI') rx_order_time +, trunc(rx.start_date) rx_start_date +, trunc(rx.end_date) rx_end_date +, to_number(rx.rx_dose_ordered) rx_dose_ordered +, rx.rx_dose_ordered_unit +, rx.rx_quantity +, 'NI' rx_dose_form +, rx.rx_refills +, rx.rx_days_supply +, rx.rx_frequency +, rx.rx_prn_flag +, rx.rx_route +, decode(rx.modifier_cd, 'MedObs:Inpatient', '01', 'MedObs:Outpatient', '02') rx_basis +, rx.rxnorm_cui +, 'OD' rx_source +, rx.rx_dispense_as_written +, rx.raw_rx_med_name +, cast(null as varchar(50)) raw_rx_frequency +, rx.raw_rxnorm_cui +, cast(null as varchar(50)) raw_rx_quantity +, cast(null as varchar(50)) raw_rx_ndc +, rx.raw_rx_dose_ordered +, rx.raw_rx_dose_ordered_unit +, rx.raw_rx_route +, cast(null as varchar(50)) raw_rx_refills + +/* ISSUE: HERON should have an actual order time. + idea: store real difference between order date start data, possibly using the update date */ +from prescribing_w_daw rx +/ + +create index prescribing_idx on prescribing (PATID, ENCOUNTERID) +/ +BEGIN +GATHER_TABLE_STATS('PRESCRIBING'); +END; +/ + +update cdm_status +set end_time = sysdate, records = (select count(*) from prescribing) +where task = 'prescribing' +/ + +select records from cdm_status where task = 'prescribing' diff --git a/Oracle/pro_cm.sql b/Oracle/pro_cm.sql new file mode 100644 index 0000000..4bef352 --- /dev/null +++ b/Oracle/pro_cm.sql @@ -0,0 +1,57 @@ +/** pro_cm - create the pro_cm table. +*/ +insert into cdm_status (task, start_time) select 'pro_cm', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE pro_cm'); +END; +/ +CREATE TABLE pro_cm( + PRO_CM_ID varchar(19) primary key, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NULL, + PRO_DATE date NOT NULL, + PRO_TIME varchar(5) NULL, + PRO_TYPE varchar(2) NULL, + PRO_ITEM_NAME varchar(50) NULL, + PRO_ITEM_LOINC varchar(10) NULL, + PRO_RESPONSE_TEXT varchar(50) NULL, + PRO_RESPONSE_NUM number(8) NOT NULL, + PRO_METHOD varchar(2) NULL, + PRO_MODE varchar(2) NULL, + PRO_CAT varchar(2) NULL, + PRO_ITEM_VERSION varchar(50) NULL, + PRO_MEASURE_NAME varchar(50) NULL, + PRO_MEASURE_SEQ varchar(50) NULL, + PRO_MEASURE_SCORE number(8) NULL, + PRO_MEASURE_THETA number(8) NULL, + PRO_MEASURE_SCALED_TSCORE number(8) NULL, + PRO_MEASURE_STANDARD_ERROR number(8) NULL, + PRO_MEASURE_COUNT_SCORED number(8) NULL, + PRO_MEASURE_LOINC varchar(10) NULL, + PRO_MEASURE_VERSION varchar(50) NULL, + PRO_ITEM_FULLNAME varchar(50) NULL, + PRO_ITEM_TEXT varchar(50) NULL, + PRO_MEASURE_FULLNAME varchar(50) NULL +) +/ + +BEGIN +PMN_DROPSQL('DROP sequence pro_cm_seq'); +END; +/ +create sequence pro_cm_seq +/ + +create or replace trigger pro_cm_trg +before insert on pro_cm +for each row +begin + select pro_cm_seq.nextval into :new.PRO_CM_ID from dual; +end; +/ +update cdm_status +set end_time = sysdate, records = (select count(*) from pro_cm) +where task = 'pro_cm' +/ +select records + 1 from cdm_status where task = 'pro_cm' diff --git a/Oracle/procedures.sql b/Oracle/procedures.sql new file mode 100644 index 0000000..fb6b36f --- /dev/null +++ b/Oracle/procedures.sql @@ -0,0 +1,71 @@ +/** procedures - create and populate the procedures table. +*/ +insert into cdm_status (task, start_time) select 'procedures', sysdate from dual +/ +BEGIN +PMN_DROPSQL('DROP TABLE procedures'); +END; +/ +CREATE TABLE procedures( + PROCEDURESID varchar(19) primary key, + PATID varchar(50) NOT NULL, + ENCOUNTERID varchar(50) NOT NULL, + ENC_TYPE varchar(2) NULL, + ADMIT_DATE date NULL, + PROVIDERID varchar(50) NULL, + PX_DATE date NULL, + PX varchar(11) NOT NULL, + PX_TYPE varchar(2) NOT NULL, + PX_SOURCE varchar(2) NULL, + PPX varchar(2) NULL, + RAW_PX varchar(50) NULL, + RAW_PX_TYPE varchar(50) NULL, + RAW_PPX varchar(50) NULL +) +/ +BEGIN +PMN_DROPSQL('DROP sequence procedures_seq'); +END; +/ +create sequence procedures_seq +/ +create or replace trigger procedures_trg +before insert on procedures +for each row +begin + select procedures_seq.nextval into :new.PROCEDURESID from dual; +end; +/ +create or replace procedure PCORNetProcedure as +begin + +PMN_DROPSQL('drop index procedures_idx'); + +execute immediate 'truncate table procedures'; + +insert into procedures(patid, encounterid, enc_type, admit_date, px_date, providerid, px, px_type, px_source) +select distinct fact.patient_num, enc.encounterid, enc.enc_type, enc.admit_date, fact.start_date, + fact.provider_id, SUBSTR(pr.pcori_basecode,INSTR(pr.pcori_basecode, ':')+1,11) px, + -- Decode can be eliminated if pcornet_proc is updated. + decode(SUBSTR(pr.c_fullname,18,2), 'HC', 'CH', SUBSTR(pr.c_fullname,18,2)) pxtype, + -- All are billing for now - see https://informatics.gpcnetwork.org/trac/Project/ticket/491 + 'BI' px_source +from i2b2fact fact + inner join pcornet_proc pr on pr.c_basecode = fact.concept_cd + inner join encounter enc on enc.patid = fact.patient_num and enc.encounterid = fact.encounter_Num +where pr.c_fullname like '\PCORI\PROCEDURE\%'; + +execute immediate 'create index procedures_idx on procedures (PATID, ENCOUNTERID)'; +GATHER_TABLE_STATS('PROCEDURES'); + +end PCORNetProcedure; +/ +BEGIN +PCORNetProcedure(); +END; +/ +update cdm_status +set end_time = sysdate, records = (select count(*) from procedures) +where task = 'procedures' +/ +select records from cdm_status where task = 'procedures' \ No newline at end of file diff --git a/Oracle/provider.sql b/Oracle/provider.sql new file mode 100644 index 0000000..ff25bce --- /dev/null +++ b/Oracle/provider.sql @@ -0,0 +1,58 @@ +/** provider - create and populate the provider table. +*/ +insert into cdm_status (task, start_time) select 'provider', sysdate from dual +/ + +BEGIN +PMN_DROPSQL('DROP TABLE provider'); +END; +/ + +CREATE TABLE provider( + PROVIDERID varchar(50) NOT NULL, + PROVIDER_SEX varchar(2) NULL, + PROVIDER_SPECIALTY_PRIMARY varchar(50) NULL, + PROVIDER_NPI NUMBER(18, 0) NULL, -- (8,0) + PROVIDER_NPI_FLAG varchar(1) NULL, + RAW_PROVIDER_SPECIALTY_PRIMARY varchar(50) NULL +) +/ + +create or replace procedure PCORNetProvider as +begin + +PMN_DROPSQL('drop index provider_idx'); + +execute immediate 'truncate table provider'; + +insert into provider(providerid, provider_sex, provider_specialty_primary, provider_npi, provider_npi_flag, raw_provider_specialty_primary) +select pd.provider_id + , case when pd.provider_sex = 'U' then 'UN' + when pd.provider_sex is null then 'NI' + else pd.provider_sex end as sex + , case when sm.npi is null then 'NI' + else nvl(sm.specialty, 'UN') end as provider_specialty_primary + , pd.provider_npi + , case when pd.provider_npi is not null then 'Y' else 'N' end as provider_npi_flag + , substr(sp.descriptive_text, 1, 50) + from &&i2b2_data_schema.provider_dimension_cdm pd + left join provider_specialty_map sm on sm.npi = pd.provider_npi + left join provider_specialty_code sp on sm.specialty = sp.code; + +execute immediate 'create index provider_idx on provider (PROVIDERID)'; +GATHER_TABLE_STATS('PROVIDER'); + +end PCORNetProvider; +/ + +begin +PCORNetProvider(); +end; +/ + +update cdm_status +set end_time = sysdate, records = (select count(*) from provider) +where task = 'provider' +/ + +select records from cdm_status where task = 'provider' diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh new file mode 100644 index 0000000..85ab0d9 --- /dev/null +++ b/Oracle/run-i2p-transform.sh @@ -0,0 +1,138 @@ +#!/bin/bash +set -e + +python load_csv.py harvest_local harvest_local.csv harvest_local.ctl pcornet_cdm_user pcornet_cdm +python load_csv.py LabNormal labnormal.csv labnormal.ctl pcornet_cdm_user pcornet_cdm + +# Run some tests +sqlplus /nolog < 0 then 1 else 1/0 end obs_fact_meds_populated from ( + select count(*) qty from observation_fact_meds + ); +EOF + + +########### Initialize Schema ########### +if [ ${initialize_schema} == 'true' ] +then + +sqlplus /nolog < 0 then 1/0 else 1 end ethnicity_update_match from ( + select * from ( + select pd.ethnicity_cd pd_code, hp.pcori_code hp_code + from "&&i2b2_data_schema".patient_dimension pd + left join hispanic_patients hp on hp.patient_num = pd.patient_num + ) + where pd_code != hp_code and not (pd_code = 'NI' and hp_code is null) + ); diff --git a/Oracle/update_to_cdm_dimensions.sh b/Oracle/update_to_cdm_dimensions.sh new file mode 100644 index 0000000..2e8ca84 --- /dev/null +++ b/Oracle/update_to_cdm_dimensions.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +# Expected environment variables (put there by Jenkins, etc.) +# export pcornet_cdm_user= +# export pcornet_cdm= +# export i2b2_data_schema= +# export i2b2_meta_schema= + +sqlplus /nolog < None: + self.setTaskStart() + self.load() + self.setTaskEnd(self.getRecordCountFromTable()) + + def load(self) -> None: + def sz(l: int, chunk: int=16) -> int: + return max(chunk, chunk * ((l + chunk - 1) // chunk)) + + db = self._dbtarget().engine + schema = MetaData() + l = list() + + with open(self.csvname) as fin: # ISSUE: ambient + dr = DictReader(fin) + + Dict # for tools that don't see type: comments. + mcl = defaultdict(int) # type: Dict[str, int] + # Iterate data in the input csv, find the largest data item + # and set the column size to the item size rounded to the nearest chunk. + for row in dr: + l.append(row) + for col in dr.fieldnames: + mcl[col] = sz(max(mcl[col], len(row[col]))) + + columns = ([Column(n, String(mcl[n])) for n in dr.fieldnames]) + table = Table(self.taskName, schema, *columns) + if table.exists(bind=db): + table.drop(db) + table.create(db) + + db.execute(table.insert(), l) diff --git a/curated_data/harvest_local.csv b/curated_data/harvest_local.csv new file mode 100644 index 0000000..4f16835 --- /dev/null +++ b/curated_data/harvest_local.csv @@ -0,0 +1,2 @@ +DATAMART_CLAIMS,DATAMART_EHR,BIRTH_DATE_MGMT,ENR_START_DATE_MGMT,ENR_END_DATE_MGMT,ADMIT_DATE_MGMT,DISCHARGE_DATE_MGMT,DX_DATE_MGMT,PX_DATE_MGMT,RX_ORDER_DATE_MGMT,RX_START_DATE_MGMT,RX_END_DATE_MGMT,DISPENSE_DATE_MGMT,LAB_ORDER_DATE_MGMT,SPECIMEN_DATE_MGMT,RESULT_DATE_MGMT,MEASURE_DATE_MGMT,ONSET_DATE_MGMT,REPORT_DATE_MGMT,RESOLVE_DATE_MGMT,PRO_DATE_MGMT,DEATH_DATE_MGMT,MEDADMIN_START_DATE_MGMT,MEDADMIN_STOP_DATE_MGMT,OBSCLIN_DATE_MGMT,OBSGEN_DATE_MGMT,ADDRESS_PERIOD_START_MGMT,ADDRESS_PERIOD_END_MGMT,VX_RECORD_DATE_MGMT,VX_ADMIN_DATE_MGMT,VX_EXP_DATE_MGMT +1,2,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01,01 diff --git a/curated_data/labnormal.csv b/curated_data/labnormal.csv new file mode 100644 index 0000000..b6f18ee --- /dev/null +++ b/curated_data/labnormal.csv @@ -0,0 +1,237 @@ +"CONCEPT_CD","LAB_NAME","SEX","AGE_LOWER","AGE_UPPER","REF_LO","REF_HI" +"KUH|COMPONENT_ID:2070","LDH TOTAL","F",0,30,600,1395 +"KUH|COMPONENT_ID:2070","LDH TOTAL","F",30,512,600,1350 +"KUH|COMPONENT_ID:2070","LDH TOTAL","F",512,2816,495,1290 +"KUH|COMPONENT_ID:2070","LDH TOTAL","F",2816,4352,405,930 +"KUH|COMPONENT_ID:2070","LDH TOTAL","F",4352,36500,300,600 +"KUH|COMPONENT_ID:2070","LDH TOTAL","M",0,30,600,1395 +"KUH|COMPONENT_ID:2070","LDH TOTAL","M",30,512,600,1350 +"KUH|COMPONENT_ID:2070","LDH TOTAL","M",512,2816,495,1290 +"KUH|COMPONENT_ID:2070","LDH TOTAL","M",2816,4352,405,930 +"KUH|COMPONENT_ID:2070","LDH TOTAL","M",4352,36500,300,600 +"KUH|COMPONENT_ID:2321","LDL","F",0,36500,,100 +"KUH|COMPONENT_ID:2321","LDL","M",0,36500,,100 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","F",0,30,24,48 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","F",30,1536,35,71 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","F",1536,36500,24,44 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","M",0,30,24,48 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","M",30,1536,35,71 +"KUH|COMPONENT_ID:3028","LYMPHOCYTES","M",1536,36500,24,44 +"KUH|COMPONENT_ID:2020","MAGNESIUM","F",0,36500,1.6,2.6 +"KUH|COMPONENT_ID:2020","MAGNESIUM","M",0,36500,1.6,2.6 +"KUH|COMPONENT_ID:3043","MCH","F",0,36500,26,34 +"KUH|COMPONENT_ID:3043","MCH","M",0,36500,26,34 +"KUH|COMPONENT_ID:3044","MCHC","F",0,36500,32,36 +"KUH|COMPONENT_ID:3044","MCHC","M",0,36500,32,36 +"KUH|COMPONENT_ID:3042","MCV","F",0,36500,80,100 +"KUH|COMPONENT_ID:3042","MCV","M",0,36500,80,100 +"KUH|COMPONENT_ID:3030","MONOCYTES","F",0,36500,4,12 +"KUH|COMPONENT_ID:3030","MONOCYTES","M",0,36500,4,12 +"KUH|COMPONENT_ID:3045","MPV (PLATELET)","F",0,36500,7,11 +"KUH|COMPONENT_ID:3045","MPV (PLATELET)","M",0,36500,7,11 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","F",0,30,43,80 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","F",30,1536,28,56 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","F",1536,36500,41,77 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","M",0,30,43,80 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","M",30,1536,28,56 +"KUH|COMPONENT_ID:3010","NEUTROPHILS","M",1536,36500,41,77 +"KUH|COMPONENT_ID:3026","NEUTROPHILS/100 LEUKOCYTES, SEG","F",0,36500,0,25 +"KUH|COMPONENT_ID:3026","NEUTROPHILS/100 LEUKOCYTES, SEG","M",0,36500,0,25 +"KUH|COMPONENT_ID:4003","PCO2, ART POC","F",0,14,27,40 +"KUH|COMPONENT_ID:4003","PCO2, ART POC","F",14,36500,35,45 +"KUH|COMPONENT_ID:4003","PCO2, ART POC","M",0,14,27,40 +"KUH|COMPONENT_ID:4003","PCO2, ART POC","M",14,36500,35,45 +"KUH|COMPONENT_ID:4001","PH, ART POC","F",0,1,7.29,7.45 +"KUH|COMPONENT_ID:4001","PH, ART POC","F",1,36500,7.35,7.45 +"KUH|COMPONENT_ID:4001","PH, ART POC","M",0,1,7.29,7.45 +"KUH|COMPONENT_ID:4001","PH, ART POC","M",1,36500,7.35,7.45 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","F",0,30,4,9 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","F",30,256,4,6 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","F",256,4608,3,5 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","F",4608,36500,2,4 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","M",0,30,4,9 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","M",30,256,4,6 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","M",256,4608,3,5 +"KUH|COMPONENT_ID:2033","PHOSPHORUS","M",4608,36500,2,4 +"KUH|COMPONENT_ID:3053","PLATELET COUNT","F",0,36500,150,400 +"KUH|COMPONENT_ID:3008","PLATELET COUNT","F",0,36500,150,400 +"KUH|COMPONENT_ID:3053","PLATELET COUNT","M",0,36500,150,400 +"KUH|COMPONENT_ID:3008","PLATELET COUNT","M",0,36500,150,400 +"KUH|COMPONENT_ID:4005","PO2, ART POC","F",0,1,54,90 +"KUH|COMPONENT_ID:4005","PO2, ART POC","F",1,36500,80,100 +"KUH|COMPONENT_ID:4005","PO2, ART POC","M",0,1,54,90 +"KUH|COMPONENT_ID:4005","PO2, ART POC","M",1,36500,80,100 +"KUH|COMPONENT_ID:2002","POTASSIUM","F",0,36500,3.5,5.1 +"KUH|COMPONENT_ID:2002","POTASSIUM","M",0,36500,3.5,5.1 +"KUH|COMPONENT_ID:3085","PTT","F",0,36500,24,40 +"KUH|COMPONENT_ID:3085","PTT","M",0,36500,24,40 +"KUH|COMPONENT_ID:3041","RBC COUNT","F",0,30,4.7,5.9 +"KUH|COMPONENT_ID:3041","RBC COUNT","F",30,3584,3.3,5.2 +"KUH|COMPONENT_ID:3041","RBC COUNT","F",3584,36500,4,5 +"KUH|COMPONENT_ID:3041","RBC COUNT","M",0,30,4.7,5.9 +"KUH|COMPONENT_ID:3041","RBC COUNT","M",30,3584,3.3,5.2 +"KUH|COMPONENT_ID:3041","RBC COUNT","M",3584,36500,4.4,5.5 +"KUH|COMPONENT_ID:3046","RDW","F",0,36500,11,15 +"KUH|COMPONENT_ID:3046","RDW","M",0,36500,11,15 +"KUH|COMPONENT_ID:2000","SODIUM","F",0,14,134,144 +"KUH|COMPONENT_ID:2000","SODIUM","F",14,256,137,147 +"KUH|COMPONENT_ID:2000","SODIUM","F",256,36500,137,147 +"KUH|COMPONENT_ID:2000","SODIUM","M",0,14,134,144 +"KUH|COMPONENT_ID:2000","SODIUM","M",14,256,137,147 +"KUH|COMPONENT_ID:2000","SODIUM","M",256,36500,137,147 +"KUH|COMPONENT_ID:7050","SQUAMOUS EPI CELLS","F",0,36500,0,5 +"KUH|COMPONENT_ID:7050","SQUAMOUS EPI CELLS","M",0,36500,0,5 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","F",0,30,4.6,7 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","F",30,256,4.6,7.5 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","F",256,36500,6,8 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","M",0,30,4.6,7 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","M",30,256,4.6,7.5 +"KUH|COMPONENT_ID:2032","TOTAL PROTEIN","M",256,36500,6,8 +"KUH|COMPONENT_ID:2319","TRIGLYCERIDE","F",0,36500,,150 +"KUH|COMPONENT_ID:2319","TRIGLYCERIDE","M",0,36500,,150 +"KUH|COMPONENT_ID:2326","TROPONIN I","F",0,36500,0,0.05 +"KUH|COMPONENT_ID:2326","TROPONIN I","M",0,36500,0,0.05 +"KUH|COMPONENT_ID:2198","TSH","F",0,36500,0.35,5 +"KUH|COMPONENT_ID:2198","TSH","M",0,36500,0.35,5 +"KUH|COMPONENT_ID:7006","URINE MACROSCOPIC, GLUCOSE","F",0,36500,, +"KUH|COMPONENT_ID:7006","URINE MACROSCOPIC, GLUCOSE","M",0,36500,, +"KUH|COMPONENT_ID:7015","URINE MACROSCOPIC, LEUKOCYTES","F",0,36500,, +"KUH|COMPONENT_ID:7015","URINE MACROSCOPIC, LEUKOCYTES","M",0,36500,, +"KUH|COMPONENT_ID:7014","URINE MACROSCOPIC, NITRITE","F",0,36500,, +"KUH|COMPONENT_ID:7014","URINE MACROSCOPIC, NITRITE","M",0,36500,, +"KUH|COMPONENT_ID:7004","URINE MACROSCOPIC, PROTEIN","F",0,36500,, +"KUH|COMPONENT_ID:7004","URINE MACROSCOPIC, PROTEIN","M",0,36500,, +"KUH|COMPONENT_ID:7009","URINE MACROSCOPIC, URINE BILE","F",0,36500,, +"KUH|COMPONENT_ID:7009","URINE MACROSCOPIC, URINE BILE","M",0,36500,, +"KUH|COMPONENT_ID:7011","URINE MACROSCOPIC, URINE BLOOD","F",0,36500,, +"KUH|COMPONENT_ID:7011","URINE MACROSCOPIC, URINE BLOOD","M",0,36500,, +"KUH|COMPONENT_ID:7008","URINE MACROSCOPIC, URINE KETONE","F",0,36500,, +"KUH|COMPONENT_ID:7008","URINE MACROSCOPIC, URINE KETONE","M",0,36500,, +"KUH|COMPONENT_ID:7003","URINE MACROSCOPIC, URINE PH","F",0,36500,5,8 +"KUH|COMPONENT_ID:7003","URINE MACROSCOPIC, URINE PH","M",0,36500,5,8 +"KUH|COMPONENT_ID:7002","URINE MACROSCOPIC, URINE SPEC GRAVITY","F",0,36500,1,1.04 +"KUH|COMPONENT_ID:7002","URINE MACROSCOPIC, URINE SPEC GRAVITY","M",0,36500,1,1.04 +"KUH|COMPONENT_ID:7012","URINE MACROSCOPIC, UROBILINOGEN","F",0,36500,, +"KUH|COMPONENT_ID:7012","URINE MACROSCOPIC, UROBILINOGEN","M",0,36500,, +"KUH|COMPONENT_ID:7017","URINE MICROSCOPIC, RBC","F",0,36500,0,3 +"KUH|COMPONENT_ID:7017","URINE MICROSCOPIC, RBC","M",0,36500,0,3 +"KUH|COMPONENT_ID:3009","WBC COUNT","F",0,30,5,20 +"KUH|COMPONENT_ID:3009","WBC COUNT","F",30,512,5,17 +"KUH|COMPONENT_ID:3009","WBC COUNT","F",512,4096,4.5,13 +"KUH|COMPONENT_ID:3009","WBC COUNT","F",4096,36500,4.5,11 +"KUH|COMPONENT_ID:3009","WBC COUNT","M",0,30,5,20 +"KUH|COMPONENT_ID:3009","WBC COUNT","M",30,512,5,17 +"KUH|COMPONENT_ID:3009","WBC COUNT","M",512,4096,4.5,13 +"KUH|COMPONENT_ID:3009","WBC COUNT","M",4096,36500,4.5,11 +"KUH|COMPONENT_ID:7016","WBC, URINE","F",0,36500,0,2 +"KUH|COMPONENT_ID:7016","WBC, URINE","M",0,36500,0,2 +"KUH|COMPONENT_ID:3027","WHITE BLOOD COUNT DIFFERENTIAL, BAND NEUTROPHIL","F",0,36500,0,10 +"KUH|COMPONENT_ID:3027","WHITE BLOOD COUNT DIFFERENTIAL, BAND NEUTROPHIL","M",0,36500,0,10 +"KUH|COMPONENT_ID:3023","WHITE BLOOD COUNT DIFFERENTIAL, BASOPHILS","F",0,36500,0,2 +"KUH|COMPONENT_ID:3023","WHITE BLOOD COUNT DIFFERENTIAL, BASOPHILS","M",0,36500,0,2 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","F",0,30,24,48 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","F",30,1536,35,71 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","F",1536,36500,24,44 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","M",0,30,24,48 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","M",30,1536,35,71 +"KUH|COMPONENT_ID:3014","WHITE BLOOD COUNT DIFFERENTIAL, LYMPHOCYTES","M",1536,36500,24,44 +"KUH|COMPONENT_ID:3025","WHITE CELL DIFFERENTIAL, ABSOLUTE BASO COUNT","F",4352,36500,0,0.2 +"KUH|COMPONENT_ID:3025","WHITE CELL DIFFERENTIAL, ABSOLUTE BASO COUNT","M",4352,36500,0,0.2 +"KUH|COMPONENT_ID:3022","WHITE CELL DIFFERENTIAL, ABSOLUTE EOS COUNT","F",4352,36500,0,0.45 +"KUH|COMPONENT_ID:3022","WHITE CELL DIFFERENTIAL, ABSOLUTE EOS COUNT","M",4352,36500,0,0.45 +"KUH|COMPONENT_ID:3016","WHITE CELL DIFFERENTIAL, ABSOLUTE LYMPH COUNT","F",4352,36500,1,4.8 +"KUH|COMPONENT_ID:3016","WHITE CELL DIFFERENTIAL, ABSOLUTE LYMPH COUNT","M",4352,36500,1,4.8 +"KUH|COMPONENT_ID:3019","WHITE CELL DIFFERENTIAL, ABSOLUTE MONO COUNT","F",4352,36500,0,0.8 +"KUH|COMPONENT_ID:3019","WHITE CELL DIFFERENTIAL, ABSOLUTE MONO COUNT","M",4352,36500,0,0.8 +"KUH|COMPONENT_ID:3012","WHITE CELL DIFFERENTIAL, ABSOLUTE NEUTROPHIL","F",4352,36500,1.8,7 +"KUH|COMPONENT_ID:3012","WHITE CELL DIFFERENTIAL, ABSOLUTE NEUTROPHIL","M",4352,36500,1.8,7 +"KUH|COMPONENT_ID:2023","ALBUMIN","F",0,36500,3.5,5 +"KUH|COMPONENT_ID:2023","ALBUMIN","M",0,36500,3.5,5 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",0,30,, +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",30,512,95,327 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",512,2560,99,232 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","F",2560,36500,25,110 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",0,30,, +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",30,512,95,327 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",512,2560,99,232 +"KUH|COMPONENT_ID:2071","ALK PHOSPHATASE","M",2560,36500,25,110 +"KUH|COMPONENT_ID:2065","ALT","F",0,256,4,76 +"KUH|COMPONENT_ID:2065","ALT","F",256,36500,7,56 +"KUH|COMPONENT_ID:2065","ALT","M",0,256,4,76 +"KUH|COMPONENT_ID:2065","ALT","M",256,36500,7,56 +"KUH|COMPONENT_ID:2006","ANION GAP","F",0,36500,3,12 +"KUH|COMPONENT_ID:2006","ANION GAP","M",0,36500,3,12 +"KUH|COMPONENT_ID:2064","AST","F",0,256,20,100 +"KUH|COMPONENT_ID:2064","AST","F",256,36500,7,40 +"KUH|COMPONENT_ID:2064","AST","M",0,256,20,100 +"KUH|COMPONENT_ID:2064","AST","M",256,36500,7,40 +"KUH|COMPONENT_ID:4007","BICARB, ART(CAL)","F",0,36500,21,28 +"KUH|COMPONENT_ID:4007","BICARB, ART(CAL)","M",0,36500,21,28 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",0,1,,2 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",1,3,,8 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",3,5,,12 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",5,6,0.3,1.2 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","F",6,36500,0.3,1.2 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",0,1,,2 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",1,3,,8 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",3,5,,12 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",5,6,0.3,1.2 +"KUH|COMPONENT_ID:2024","BILIRUBIN, TOTAL BILIRUBIN","M",6,36500,0.3,1.2 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","F",0,256,4,12 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","F",256,4608,5,20 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","F",4608,36500,7,25 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","M",0,256,4,12 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","M",256,4608,5,20 +"KUH|COMPONENT_ID:2007","BLOOD UREA NITROGEN, BUN","M",4608,36500,7,25 +"KUH|COMPONENT_ID:2017","CALCIUM","F",0,36500,8.5,10.6 +"KUH|COMPONENT_ID:2017","CALCIUM","M",0,36500,8.5,10.6 +"KUH|COMPONENT_ID:2018","CALCIUM, IONIZED POC","F",0,36500,1,1.3 +"KUH|COMPONENT_ID:2018","CALCIUM, IONIZED POC","M",0,36500,1,1.3 +"KUH|COMPONENT_ID:2004","CHLORIDE","F",0,36500,98,110 +"KUH|COMPONENT_ID:2004","CHLORIDE","M",0,36500,98,110 +"KUH|COMPONENT_ID:2318","CHOLESTEROL","F",0,36500,,200 +"KUH|COMPONENT_ID:2318","CHOLESTEROL","M",0,36500,,200 +"KUH|COMPONENT_ID:2005","CO2","F",0,14,19,24 +"KUH|COMPONENT_ID:2005","CO2","F",14,4608,20,28 +"KUH|COMPONENT_ID:2005","CO2","F",4608,36500,21,30 +"KUH|COMPONENT_ID:2005","CO2","M",0,14,19,24 +"KUH|COMPONENT_ID:2005","CO2","M",14,4608,20,28 +"KUH|COMPONENT_ID:2005","CO2","M",4608,36500,21,30 +"KUH|COMPONENT_ID:2009","CREATININE","F",0,30,0.3,1 +"KUH|COMPONENT_ID:2009","CREATININE","F",30,256,0.2,0.4 +"KUH|COMPONENT_ID:2009","CREATININE","F",256,4608,0.3,1 +"KUH|COMPONENT_ID:2009","CREATININE","F",4608,36500,0.4,1 +"KUH|COMPONENT_ID:2009","CREATININE","M",0,30,0.3,1 +"KUH|COMPONENT_ID:2009","CREATININE","M",30,256,0.2,0.4 +"KUH|COMPONENT_ID:2009","CREATININE","M",256,4608,0.3,1 +"KUH|COMPONENT_ID:2009","CREATININE","M",4608,36500,0.4,1.24 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","F",0,1,45,80 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","F",1,30,50,80 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","F",30,36500,70,100 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","M",0,1,45,80 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","M",1,30,50,80 +"KUH|COMPONENT_ID:2012","GLUCOSE, POC","M",30,36500,70,100 +"KUH|COMPONENT_ID:2320","HDL","F",0,36500,40, +"KUH|COMPONENT_ID:2320","HDL","M",0,36500,40, +"KUH|COMPONENT_ID:3004","HEMATOCRIT","F",0,30,51,65 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","F",30,180,38,50 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","F",180,1024,30,42 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","F",1024,3584,35,46 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","F",3584,36500,36,45 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","M",0,30,51,65 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","M",30,180,38,50 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","M",180,1024,30,42 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","M",1024,3584,36,46 +"KUH|COMPONENT_ID:3004","HEMATOCRIT","M",3584,36500,40,50 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",0,30,17.5,21.5 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",30,180,12.8,15.5 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",180,1024,10,14 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",1024,3584,11,14 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","F",3584,36500,12,15 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",0,30,17.5,21.5 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",30,180,12.8,15.5 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",180,1024,10,14 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",1024,3584,11,14 +"KUH|COMPONENT_ID:3000","HEMOGLOBIN","M",3584,36500,13.5,16.5 +"KUH|COMPONENT_ID:2034","HEMOGLOBIN A1c","F",0,36500,4,6 +"KUH|COMPONENT_ID:2034","HEMOGLOBIN A1c","M",0,36500,4,6 diff --git a/curated_data/language.csv b/curated_data/language.csv new file mode 100644 index 0000000..945c5c7 --- /dev/null +++ b/curated_data/language.csv @@ -0,0 +1,428 @@ +CODE,DESCRIPTIVE_TEXT +AAR,Afar +ABK,Abkhazian +ACE,Achinese +ACH,Acoli +ADA,Adangme +ADY,Adyghe +ADY,Adygei +AFR,Afrikaans +AIN,Ainu +AKA,Akan +ALE,Aleut +ALT,Southern Altai +AMH,Amharic +ANP,Angika +ARA,Arabic +ARG,Aragonese +ARN,Mapudungun +ARN,Mapuche +ARP,Arapaho +ARW,Arawak +ASM,Assamese +AST,Asturian +AST,Bable +AST,Leonese +AST,Asturleonese +AVA,Avaric +AWA,Awadhi +AYM,Aymara +AZE,Azerbaijani +BAK,Bashkir +BAL,Baluchi +BAM,Bambara +BAN,Balinese +BAS,Basa +BEJ,Beja +BEJ,Bedawiyet +BEL,Belarusian +BEM,Bemba +BEN,Bengali +BHO,Bhojpuri +BIK,Bikol +BIN,Bini +BIN,Edo +BIS,Bislama +BLA,Siksika +BOD,Tibetan +BOS,Bosnian +BRA,Braj +BRE,Breton +BUA,Buriat +BUG,Buginese +BUL,Bulgarian +BYN,Bilin +BYN,Blin +CAD,Caddo +CAR,Galibi Carib +CAT,Catalan +CAT,Valencian +CEB,Cebuano +CES,Czech +CHA,Chamorro +CHE,Chechen +CHK,Chuukese +CHM,Mari +CHN,Chinook jargon +CHO,Choctaw +CHP,Chipewyan +CHP,Dene Suline +CHR,Cherokee +CHV,Chuvash +CHY,Cheyenne +COR,Cornish +COS,Corsican +CRE,Cree +CRH,Crimean Tatar +CRH,Crimean Turkish +CSB,Kashubian +CYM,Welsh +DAK,Dakota +DAN,Danish +DAR,Dargwa +DEL,Delaware +DEN,Slave (Athapascan) +DEU,German +DGR,Dogrib +DIN,Dinka +DIV,Dhivehi +DIV,Dhivehi +DIV,Maldivian +DOI,Dogri +DSB,Lower Sorbian +DUA,Duala +DYU,Dyula +DZO,Dzongkha +EFI,Efik +EKA,Ekajuk +ELL,Modern Greek (1453-) +ENG,English +EST,Estonian +EUS,Basque +EWE,Ewe +EWO,Ewondo +FAN,Fang +FAO,Faroese +FAS,Persian +FAT,Fanti +FIJ,Fijian +FIL,Filipino +FIL,Pilipino +FIN,Finnish +FON,Fon +FRA,French +FRR,Northern Frisian +FRS,Eastern Frisian +FRY,Western Frisian +FUL,Fulah +FUR,Friulian +GAA,Ga +GAY,Gayo +GBA,Gbaya +GIL,Gilbertese +GLA,Gaelic +GLA,Scottish Gaelic +GLE,Irish +GLG,Galician +GLV,Manx +GON,Gondi +GOR,Gorontalo +GRB,Grebo +GRN,Guarani +GSW,Swiss German +GSW,Alemannic +GSW,Alsatian +GUJ,Gujarati +GWI,Gwich'in +HAI,Haida +HAT,Haitian +HAT,Haitian Creole +HAU,Hausa +HAW,Hawaiian +HEB,Hebrew +HER,Herero +HIL,Hiligaynon +HIN,Hindi +HMN,Hmong +HMN,Mong +HMO,Hiri Motu +HRV,Croatian +HSB,Upper Sorbian +HUN,Hungarian +HUP,Hupa +HYE,Armenian +IBA,Iban +IBO,Igbo +III,Sichuan Yi +III,Nuosu +IKU,Inuktitut +ILO,Iloko +IND,Indonesian +INH,Ingush +IPK,Inupiaq +ISL,Icelandic +ITA,Italian +JAV,Javanese +JPN,Japanese +JPR,Judeo-Persian +JRB,Judeo-Arabic +KAA,Kara-Kalpak +KAB,Kabyle +KAC,Kachin +KAC,Jingpho +KAL,Kalaallisut +KAL,Greenlandic +KAM,Kamba +KAN,Kannada +KAS,Kashmiri +KAT,Georgian +KAU,Kanuri +KAZ,Kazakh +KBD,Kabardian +KHA,Khasi +KHM,Central Khmer +KIK,Kikuyu +KIK,Gikuyu +KIN,Kinyarwanda +KIR,Kirghiz +KIR,Kyrgyz +KMB,Kimbundu +KOK,Konkani +KOM,Komi +KON,Kongo +KOR,Korean +KOS,Kosraean +KPE,Kpelle +KRC,Karachay-Balkar +KRL,Karelian +KRU,Kurukh +KUA,Kuanyama +KUA,Kwanyama +KUM,Kumyk +KUR,Kurdish +KUT,Kutenai +LAD,Ladino +LAH,Lahnda +LAM,Lamba +LAO,Lao +LAV,Latvian +LEZ,Lezghian +LIM,Limburgan +LIM,Limburger +LIM,Limburgish +LIN,Lingala +LIT,Lithuanian +LOL,Mongo +LOZ,Lozi +LTZ,Luxembourgish +LTZ,Letzeburgesch +LUA,Luba-Lulua +LUB,Luba-Katanga +LUG,Ganda +LUI,Luiseno +LUN,Lunda +LUO,Luo (Kenya and Tanzania) +LUS,Lushai +MAD,Madurese +MAG,Magahi +MAH,Marshallese +MAI,Maithili +MAK,Makasar +MAL,Malayalam +MAN,Mandingo +MAR,Marathi +MAS,Masai +MDF,Moksha +MDR,Mandar +MEN,Mende +MIC,Mi'kmaq +MIC,Micmac +MIN,Minangkabau +MKD,Macedonian +MLG,Malagasy +MLT,Maltese +MNC,Manchu +MNI,Manipuri +MOH,Mohawk +MON,Mongolian +MOS,Mossi +MRI,Maori +MSA,Malay +MUS,Creek +MWL,Mirandese +MWR,Marwari +MYA,Burmese +MYV,Erzya +NAP,Neapolitan +NAU,Nauru +NAV,Navajo +NAV,Navaho +NBL,South Ndebele +NDE,North Ndebele +NDO,Ndonga +NDS,Low German +NDS,Low Saxon +NEP,Nepali +NEW,Nepal Bhasa +NEW,Newari +NIA,Nias +NIU,Niuean +NLD,Dutch +NLD,Flemish +NNO,Norwegian Nynorsk +NOB,Norwegian Bokmal +NOG,Nogai +NOR,Norwegian +NQO,N'Ko +NSO,Pedi +NSO,Sepedi +NSO,Northern Sotho +NYA,Chichewa +NYA,Chewa +NYA,Nyanja +NYM,Nyamwezi +NYN,Nyankole +NYO,Nyoro +NZI,Nzima +OCI,Occitan (post 1500) +OJI,Ojibwa +ORI,Oriya +ORM,Oromo +OSA,Osage +OSS,Ossetian +OSS,Ossetic +PAG,Pangasinan +PAM,Pampanga +PAM,Kapampangan +PAN,Panjabi +PAN,Punjabi +PAP,Papiamento +PAU,Palauan +POL,Polish +PON,Pohnpeian +POR,Portuguese +PUS,Pushto +PUS,Pashto +QUE,Quechua +RAJ,Rajasthani +RAP,Rapanui +RAR,Rarotongan +RAR,Cook Islands Maori +ROH,Romansh +ROM,Romany +RON,Romanian +RON,Moldavian +RON,Moldovan +RUN,Rundi +RUP,Aromanian +RUP,Arumanian +RUP,Macedo-Romanian +RUS,Russian +SAD,Sandawe +SAG,Sango +SAH,Yakut +SAS,Sasak +SAT,Santali +SCN,Sicilian +SCO,Scots +SEL,Selkup +SHN,Shan +SID,Sidamo +SIN,Sinhala +SIN,Sinhalese +SLK,Slovak +SLV,Slovenian +SMA,Southern Sami +SME,Northern Sami +SMJ,Lule Sami +SMN,Inari Sami +SMO,Samoan +SMS,Skolt Sami +SNA,Shona +SND,Sindhi +SNK,Soninke +SOM,Somali +SOT,Southern Sotho +SPA,Spanish +SPA,Castilian +SQI,Albanian +SRD,Sardinian +SRN,Sranan Tongo +SRP,Serbian +SRR,Serer +SSW,Swati +SUK,Sukuma +SUN,Sundanese +SUS,Susu +SWA,Swahili +SWE,Swedish +SYR,Syriac +TAH,Tahitian +TAM,Tamil +TAT,Tatar +TEL,Telugu +TEM,Timne +TER,Tereno +TET,Tetum +TGK,Tajik +TGL,Tagalog +THA,Thai +TIG,Tigre +TIR,Tigrinya +TIV,Tiv +TKL,Tokelau +TLI,Tlingit +TMH,Tamashek +TOG,Tonga (Nyasa) +TON,Tonga (Tonga Islands) +TPI,Tok Pisin +TSI,Tsimshian +TSN,Tswana +TSO,Tsonga +TUK,Turkmen +TUM,Tumbuka +TUR,Turkish +TVL,Tuvalua +TWI,Twi +TYV,Tuvinian +UDM,Udmurt +UIG,Uighur +UIG,Uyghur +UKR,Ukrainian +UMB,Umbundu +URD,Urdu +UZB,Uzbek +VAI,Vai +VEN,Venda +VIE,Vietnamese +VOT,Votic +WAL,Wolaitta +WAL,Wolaytta +WAR,Waray +WAS,Washo +WLN,Walloon +WOL,Wolof +XAL,Kalmyk +XAL,Oirat +XHO,Xhosa +YAO,Yao +YAP,Yapese +YID,Yiddish +YOR,Yoruba +ZAP,Zapotec +ZEN,Zenaga +ZGH,Standard Moroccan Tamazight +ZHA,Zhuang +ZHA,Chuang +ZHO,Chinese +ZUL,Zulu +ZUN,Zuni +ZZA,Zaza +ZZA,Dimili +ZZA,Dimli +ZZA,Kirdki +ZZA,Kirmanjki +ZZA,Zazaki +NI,No information +UN,Unknown +OT,Other diff --git a/curated_data/payer_map.csv b/curated_data/payer_map.csv new file mode 100644 index 0000000..15f8c13 --- /dev/null +++ b/curated_data/payer_map.csv @@ -0,0 +1,322 @@ +PAYER_NAME,FINANCIAL_CLASS,CODE,DESCRIPTIVE_TEXT,SOURCE SYSTEM +AIH COLLECTIONS,@,8,NO PAYMENT from an Organization/Agency/Program/Private Payer Listed,IDX +BANKRUPTCY,@,83,Refusal to Pay/Bad Debt,IDX +BERLIN WHEELER COLLECTIONS,@,8,NO PAYMENT from an Organization/Agency/Program/Private Payer Listed,IDX +BERMAN & RABIN COLLECTIONS,@,8,NO PAYMENT from an Organization/Agency/Program/Private Payer Listed,IDX +BUDGET PLAN,@,OT,Other,IDX +COLLECTION AGENCY,@,8,NO PAYMENT from an Organization/Agency/Program/Private Payer Listed,IDX +CONSUMER COLL MGMT COLLECTIONS,@,OT,Other,IDX +CREDIT LETTER SENT,@,OT,Other,IDX +CRIME VICTIMS KS,@,37,Local Government,IDX +CRIME VICTIMS MO,@,37,Local Government,IDX +CSR COLLECTIONS,@,2,MEDICAID,IDX +DEPT OF SOCIAL REHAB,@,2,MEDICAID,IDX +DMERC,@,1,MEDICARE,IDX +DO NOT BILL,@,82,No Charge,IDX +DO NOT DUNN,@,82,No Charge,IDX +ELECTIVE PACKAGE RATE,@,9999,Unavailable / No Payer Specified / Blank,IDX +ELECTIVE PACKAGE RATE A,@,9999,Unavailable / No Payer Specified / Blank,IDX +GRANT/STUDY,@,823,Research/Clinical Trial,IDX +LITIGATION/LEGAL,@,97,Legal Liability / Liability Insurance,IDX +LIVING DONOR BALANCE,@,85,Research/Donor,IDX +MUSCULAR DYSTROPHY ASSOC,@,8,NO PAYMENT from an Organization/Agency/Program/Private Payer Listed,IDX +Other NOS,@,9999,MISCELLANEOUS/OTHER,UHC +PATIENT REQUEST-DO NOT BILL INSURANCE,@,OT,Other,IDX +PER SE TURN LIST REVIEW,@,OT,Other,IDX +RECONCILIATION,@,OT,Other,IDX +RETURN MAIL/BAD ADDRESS,@,OT,Other,IDX +RTN MAIL AFTER TURN LIST RVW,@,OT,Other,IDX +SUSPENDED CLAIMS,@,OT,Other,IDX +TURN LIST/CREDIT REVIEW,@,OT,Other,IDX +Unknown NOS,@,9999,NO PAYMENT from an Organization/Agency/Program/Private Payer Listed,UHC +AMERIGROUP KS A,@,2,MEDICAID,IDX +AMERIGROUP KS B,@,2,MEDICAID,IDX +FAMILY HEALTH PARTNERS,@,1,MEDICARE,IDX +GLOBAL TRANSPLANTS,@,OT,Other,IDX +KANCARE BEHAVIORAL HLTH A,@,2,MEDICAID,IDX +KANCARE BEHAVIORAL HLTH B,@,2,MEDICAID,IDX +SUNFLOWER STATE KS A,@,2,MEDICAID,IDX +SUNFLOWER STATE KS B,@,2,MEDICAID,IDX +AETNA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +AETNA MEDICAID OOS,Medicaid Repl OOS,25,Medicaid - Out of State,EPIC +AETNA MEDICARE,Medicare Repl,1,MEDICARE,EPIC +AETNA USHC HMO/POS A,@,5,PRIVATE HEALTH INSURANCE,IDX +AETNA USHC HMO/POS B,@,5,PRIVATE HEALTH INSURANCE,IDX +AETNA USHC PPO/TRAD A,@,5,PRIVATE HEALTH INSURANCE,IDX +AETNA USHC PPO/TRAD B,@,5,PRIVATE HEALTH INSURANCE,IDX +AGENCY,Other,349,Other,EPIC +AHA-HEALTHCARE PREFERRED,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +ALLEGIANCE BENEFIT PLAN MGMT INC,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +ALLWELL (OON),Medicare Repl,1,MEDICARE,EPIC +AMBETTER,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +AMERICAN CONTINENTAL INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +AMERICAN NATIONAL LIFE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +AMERICAN REPUBLIC INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +AMERIGROUP MEDICAID KS,Medicaid Repl KS,2,MEDICAID,EPIC +ASSURANT HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +BAKERY & CONFECTIONARY UNION,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +BANKERS LIFE & CASUALTY,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +BCBS KANSAS,BCBS,6,BLUE CROSS/BLUE SHIELD,EPIC +BCBS KC,BCBS,6,BLUE CROSS/BLUE SHIELD,EPIC +BCBS KC MEDICARE,Medicare Repl,1,MEDICARE,EPIC +BCBS KC APPEALS-KUPI,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC APPEALS-PER SE,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC HMO/POS A,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC HMO/POS B,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC INDEMNITY/OTHER,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC PPO/TRAD A,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC PPO/TRAD B,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KC PREFERRED CARE OTHER,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KS APPEALS-KUPI,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KS APPEALS-PER SE,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KS HMO/POS A,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KS HMO/POS B,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KS PPO/TRAD A,@,6,BLUE CROSS/BLUE SHIELD,IDX +BCBS KS PPO/TRAD B,@,6,BLUE CROSS/BLUE SHIELD,IDX +BENEFIT ADMIN SYSTEM,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +BENEFIT MANAGEMENT INC,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +BOILERMAKERS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CARE,Medicare,1,MEDICARE,EPIC +CARE IMPROVEMENT PLUS,Medicare Repl,1,MEDICARE,EPIC +CARE SUPPLEMENT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CENTENE MEDICAID KS,Medicaid Repl KS,2,MEDICAID,EPIC +CENTRAL RESERVE LIFE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CENTURY HEALTH SOLUTIONS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CERNER,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CHAMPVA,VA,3221,Civilian Health and Medical Program for the VA (CHAMPVA),EPIC +CHAMPVA,Commercial,3221,Civilian Health and Medical Program for the VA (CHAMPVA),EPIC +CHESTERFIELD RESOURCES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CHRISTIAN FIDELITY,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CIGNA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CIGNA MEDICARE,Medicare Repl,1,MEDICARE,EPIC +CIGNA HMO/POS A,@,5,PRIVATE HEALTH INSURANCE,IDX +CIGNA HMO/POS B,@,5,PRIVATE HEALTH INSURANCE,IDX +CIGNA PPO/TRAD A,@,5,PRIVATE HEALTH INSURANCE,IDX +CIGNA PPO/TRAD B,@,5,PRIVATE HEALTH INSURANCE,IDX +CONSOLIDATED BILLING,Medicare Repl,1,MEDICARE,EPIC +CONTINENTAL GENERAL INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CONTINENTAL LIFE OF BRENTWOOD,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CORESOURCE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CORIZON,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CORRECT CARE SOLUTIONS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +COVENTRY,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +COVENTRY MEDICAID OOS,Medicaid Repl OOS,25,Medicaid - Out of State,EPIC +COVENTRY MEDICARE,Medicare Repl,1,MEDICARE,EPIC +COVENTRY/PRINCIPAL HMO/POS A,@,5,PRIVATE HEALTH INSURANCE,IDX +COVENTRY/PRINCIPAL HMO/POS B,@,5,PRIVATE HEALTH INSURANCE,IDX +COVENTRY/PRINCIPAL PPO A,@,5,PRIVATE HEALTH INSURANCE,IDX +COVENTRY/PRINCIPAL PPO B,@,5,PRIVATE HEALTH INSURANCE,IDX +CRIME VICTIM KS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +CRIME VICTIM MO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +DISABILITY DETERM,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +EMPIRE PLAN/NYSHIP,BCBS,6,BLUE CROSS/BLUE SHIELD,EPIC +EMPLOYERS HEALTH NETWORK,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +EQUITABLE LIFE & CASUALTY,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +FARMERS INSURANCE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +FIRST HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +FRA MILICARE PLUS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +GALLAGHER BASSETT,Worker's Comp,95,Worker's Compensation,EPIC +GEHA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +GENERIC COMMERCIAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +GENERIC TRANSPLANT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +GPA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +HCH ADMINISTRATION,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +HEALTH COST SOLUTIONS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +HEALTH NET FEDERAL SERVICES,Tricare,3113,TRICARE Standard - Fee For Service,EPIC +HEALTHLINK,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +HOME STATE HEALTH PLAN,Medicaid Repl OOS,25,Medicaid - Out of State,EPIC +HUMANA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +HUMANA MEDICARE,Medicare Repl,1,MEDICARE,EPIC +HUMANA MILITARY,Tricare,31,Deparment of Defense ,EPIC +HUMANA HMO A,@,5,PRIVATE HEALTH INSURANCE,IDX +HUMANA HMO B,@,5,PRIVATE HEALTH INSURANCE,IDX +HUMANA PPO A,@,5,PRIVATE HEALTH INSURANCE,IDX +HUMANA PPO B,@,5,PRIVATE HEALTH INSURANCE,IDX +IBEW LOCAL UNION 124 H&W,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +INACTIVE NEW DIRECTIONS,BCBS,6,BLUE CROSS/BLUE SHIELD,EPIC +INACTIVE-TRIWEST,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +INDIAN HEALTH SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +KANSAS CITY HOSPICE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +KENDALLWOOD HOSPICE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +KS MEDICAID,MEDICAID KS,2,MEDICAID,EPIC +KS MEDICAID PENDING,MEDICAID PENDING,NI,No Information,EPIC +KS MEDICAID HMO B,@,2,MEDICAID,IDX +KU ATHLETICS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +KU RX INTERNAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +LARNED STATE HOSPITAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +LEAGUE MEDICAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +LIFETRAC TRANSPLANT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MEDICA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MEDICAID OTHER HMO KS GENERIC,Medicaid Repl KS,2,MEDICAID,EPIC +MEDICAID OTHER HMO OOS GENERIC,Medicaid Repl OOS,25,Medicaid - Out of State,EPIC +MEDICAID OUT OF STATE,Medicaid OOS,25,Medicaid - Out of State,EPIC +MEDICAL EXCESS TRANSPLANT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MEDICAID KS,@,2,MEDICAID,IDX +MEDICAID KS APPEALS-KUPI,@,2,MEDICAID,IDX +MEDICAID KS APPEALS-PER SE,@,2,MEDICAID,IDX +MEDICAID KS HMO (102 REPLACEMENT),@,2,MEDICAID,IDX +MEDICAID MO,@,2,MEDICAID,IDX +MEDICAID MO APPEALS-KUPI,@,2,MEDICAID,IDX +MEDICAID MO APPEALS-PER SE,@,2,MEDICAID,IDX +MEDICAID MO HMO,@,2,MEDICAID,IDX +MEDICAID MO HMO B,@,2,MEDICAID,IDX +MEDICARE,Medicare,1,MEDICARE,EPIC +MEDICARE RAILROAD,Medicare,1,MEDICARE,EPIC +MEDICARE REPL GENERIC,Medicare Repl,1,MEDICARE,EPIC +MEDICARE APPEALS-KUPI,@,1,MEDICARE,IDX +MEDICARE APPEALS-PER SE,@,1,MEDICARE,IDX +MEDICARE HMO,@,1,MEDICARE,IDX +MEDICARE HMO B,@,1,MEDICARE,IDX +MEDICARE PPO A,@,1,MEDICARE,IDX +MEDICARE RR APPEALS-PER SE,@,1,MEDICARE,IDX +MEDICO INSURANCE CO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MEGA LIFE & HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MERITAIN HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +METRO CARE,Other,349,Other,EPIC +MH NET,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MISSOURI CARE,Medicaid Repl OOS,25,Medicaid - Out of State,EPIC +MO MEDICAID,Medicaid OOS,25,Medicaid - Out of State,EPIC +MO MEDICAID PENDING,MEDICAID PENDING,NI,No Information,EPIC +MOAA MEDIPLUS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MODEL MC PAYOR,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MULTIPLAN,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +MUTUAL OF OMAHA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +NATIONAL ASSN LETTER CARRIERS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +NECA IBEW FAMILY MEDICAL PLAN,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +NORTH AMERICAN INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +OLD SURETY LIFE INSURANCE CO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +OPTUMHEALTH,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +ORSCHELN INDUSTRIES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +PHP,BCBS,6,BLUE CROSS/BLUE SHIELD,EPIC +PHYSICIANS MUTUAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +PLUMBERS LOCAL 8,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +PRINCIPAL FINANCIAL GROUP,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +PRISON HEALTH SYSTEMS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +PYRAMID LIFE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +QUIK TRIP,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +REI,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RESERVE NATIONAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX ADVANCE PCS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX AETNA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX ALLWIN DATA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX ALPHASCRIP,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX ARGUS HEALTH SYSTEMS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX CATALYST,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX CIGNA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX CVS/CAREMARK,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX EMDEON,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX EMPLOYEE BENEFITS MANAGEMENT SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX ENVISIONRX,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX EXPRESS SCRIPTS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX HEALTHTRANS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX HUMANA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX LDI INTEGRATED PHARMACY SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX MAGELLAN,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX MAXOR PLUS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX MEDCO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX MEDIMPACT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX MEDTRAK,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX MO MEDICAID,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX NATIONAL PRESCRIPTION SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX NETCARD SYSTEMS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX OPTUM RX,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX PBM PLUS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX PHARMACY DATA MANAGEMENT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX PRIME THERAPEUTICS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX PROCARE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX RESTAT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX RYAN WHITE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX SAV-RX,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX THERAPY FIRST,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX TMESYS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX UNITEDHEALTHCARE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +RX US SCRIPT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +SANFORD HEALTH PLAN,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +SEECHANGE HEALTH,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +SELMAN AND COMPANY,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +SHEET METAL WORKERS NATIONAL,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STANDARD LIFE & ACCIDENT,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STAR HRG/STAR ADMIN SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STARMARK,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STATE FARM INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STATE OF KANSAS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STATE OF MISSOURI,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STERLING LIFE INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +STUDENT ASSURANCE SERVICES,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +THREE RIVERS PROVIDER NETWORK,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +THRIVENT FOR LUTHERANS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +TODAY'S OPTIONS,Medicare Repl,1,MEDICARE,EPIC +TRANSAMERICA LIFE INS CO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +TRICARE,Tricare,3113,TRICARE Standard - Fee For Service,EPIC +TRICARE HMO,@,3111,TRICARE Prime—HMO,IDX +TRICARE PPO/TRAD,@,3112,TRICARE Extra—PPO,IDX +TRICARE UHC HMO A,@,3111,TRICARE Prime—HMO,IDX +TRICARE UHC HMO B,@,3111,TRICARE Prime—HMO,IDX +TRICARE UHC PPO A,@,3112,TRICARE Extra—PPO,IDX +TRICARE UHC PPO B,@,3112,TRICARE Extra—PPO,IDX +TRIWEST,VA,32,Department of Veterans Affairs,EPIC +UHC,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +UHC MEDICAID KS,Medicaid Repl KS,2,MEDICAID,EPIC +UHC MEDICAID MO,Medicaid Repl OOS,25,Medicaid - Out of State,EPIC +UHC MEDICARE,Medicare Repl,1,MEDICARE,EPIC +UHC COMMUNITY PLAN KS A,@,2,PRIVATE HEALTH INSURANCE,IDX +UHC COMMUNITY PLAN KS B,@,2,PRIVATE HEALTH INSURANCE,IDX +UNITED AMERICAN INS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +UNITED HEALTH/METRA HMO/POS A,@,5,PRIVATE HEALTH INSURANCE,IDX +UNITED HEALTH/METRA HMO/POS B,@,5,PRIVATE HEALTH INSURANCE,IDX +UNITED HEALTH/METRA PPO/TRAD A,@,5,PRIVATE HEALTH INSURANCE,IDX +UNITED HEALTH/METRA PPO/TRAD B,@,5,PRIVATE HEALTH INSURANCE,IDX +UPREHS,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +US MARSHALL SERVICE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +USA MANAGED CARE,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +USAA LIFE INS CO,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +VA,VA,32,Department of Veterans Affairs,EPIC +VETERANS ADMINISTRATION,VA,32,Department of Veterans Affairs,EPIC +VIA CHRISTI HOPE,Medicaid Repl KS,2,MEDICAID,EPIC +VISION,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +WILSON-MCSHANE CORP,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +WORKERS COMP,Worker's Comp,95,Worker's Compensation,EPIC +WORKMAN'S COMPENSATION,@,95,Worker's Compensation,IDX +WPPA,Commercial,5,PRIVATE HEALTH INSURANCE,EPIC +WYJO CARE,Other,349,Other,EPIC +AUTO/LIABILITY,@,9,MISCELLANEOUS/OTHER,IDX +BAD DEBT,@,83,Refusal to Pay/Bad Debt,IDX +CHARITY/HARDSHIP,@,821,Charity,IDX +DISABILITY DETERMINATIONS,@,9,MISCELLANEOUS/OTHER,IDX +KUH EMPL SUPPLEMENTAL PLAN,@,9,MISCELLANEOUS/OTHER,IDX +KUPI ADMINISTRATIVE,@,9,MISCELLANEOUS/OTHER,IDX +KUPI COMPLIANCE,@,9,MISCELLANEOUS/OTHER,IDX +LIABILITY OTHER (NOT AUTO/WC),@,9,MISCELLANEOUS/OTHER,IDX +OTHER/COMMERCIAL APPEALS-KUPI,@,5,PRIVATE HEALTH INSURANCE,IDX +OTHER/COMMERCIAL APPEALS-PER SE,@,5,PRIVATE HEALTH INSURANCE,IDX +OTHER/COMMERCIAL HMO/POS A,@,5,PRIVATE HEALTH INSURANCE,IDX +OTHER/COMMERCIAL HMO/POS B,@,5,PRIVATE HEALTH INSURANCE,IDX +OTHER/COMMERCIAL PPO/TRAD A,@,5,PRIVATE HEALTH INSURANCE,IDX +OTHER/COMMERCIAL PPO/TRAD B,@,5,PRIVATE HEALTH INSURANCE,IDX +SELF PAY,@,81,Self-pay,IDX +Medicare Traditional/Indemnity,@,12,Medicare (Non-managed Care),UHC +Commercial/Private Preferred Provider Organization (PPO),@,512,Commercial Managed Care - PPO,UHC +Medicaid/Managed Care,@,21,Medicaid (Managed Care),UHC +Medicaid Traditional/Indemnity,@,22,Medicaid (Non-managed Care Plan),UHC +Commercial/Private Health Maintenance Organization (HMO),@,511,Commercial Managed Care - HMO,UHC +Self-Pay - Uninsured NOS,@,81,Self-pay,UHC +Medicare/Managed Care,@,11,Medicare (Managed Care),UHC +Commercial/Private Traditional/Indemnity,@,52,Private Health Insurance - Indemnity,UHC +Commercial/Private University Employees,@,5,PRIVATE HEALTH INSURANCE,UHC +Military Veterans Administration,@,3,OTHER GOVERNMENT (Federal/State/Local) (excluding Department of Corrections),UHC +Military Tri-Care,@,311,TRICARE (CHAMPUS),UHC +County Medically Indigent Services NOS,@,3,OTHER GOVERNMENT (Federal/State/Local) (excluding Department of Corrections),UHC +Commercial/Private Transplant Network,@,5,PRIVATE HEALTH INSURANCE,UHC +Auto Insurance Traditional/Indemnity,@,96,Auto Insurance (includes no fault),UHC +Workers Compensation NOS,@,95,Worker's Compensation,UHC +Workers Compensation Traditional/Indemnity,@,951,Worker's Comp HMO,UHC +Charity NOS,@,821,Charity,UHC +State Assisted Healthcare NOS,@,3,OTHER GOVERNMENT (Federal/State/Local) (excluding Department of Corrections),UHC +State Assisted Healthcare Prisoners,@,4,DEPARTMENTS OF CORRECTIONS,UHC +Commercial/Private Prisoners,@,5,PRIVATE HEALTH INSURANCE,UHC +Medicare NOS,@,1,MEDICARE,UHC +Medicaid Transplant Network,@,2,MEDICAID,UHC +Research NOS,@,823,Research/Clinical Trial,UHC +Self-Pay - Cash in Full NOS,@,81,Self-pay,UHC +Commercial/Private Point-of-Service (POS),@,513,Commercial Managed Care - POS,UHC +Commercial/Private NOS,@,3,OTHER GOVERNMENT (Federal/State/Local) (excluding Department of Corrections),UHC diff --git a/curated_data/provider_specialty_code.csv b/curated_data/provider_specialty_code.csv new file mode 100644 index 0000000..73f94f4 --- /dev/null +++ b/curated_data/provider_specialty_code.csv @@ -0,0 +1,857 @@ +CODE,DESCRIPTIVE_TEXT,GROUPING +101Y00000X,Counselor,Behavioral Health & Social Service Providers +101YA0400X,Counselor Addiction Substance Use Disorder,Behavioral Health & Social Service Providers +101YM0800X,Counselor Mental Health,Behavioral Health & Social Service Providers +101YP1600X,Counselor Pastoral,Behavioral Health & Social Service Providers +101YP2500X,Counselor Professional,Behavioral Health & Social Service Providers +101YS0200X,Counselor School,Behavioral Health & Social Service Providers +102L00000X,Psychoanalyst ,Behavioral Health & Social Service Providers +102X00000X,Poetry Therapist ,Behavioral Health & Social Service Providers +103G00000X,Clinical Neuropsychologist ,Behavioral Health & Social Service Providers +103GC0700X,Clinical Neuropsychologist Clinical,Behavioral Health & Social Service Providers +103K00000X,Behavioral Analyst ,Behavioral Health & Social Service Providers +103T00000X,Psychologist ,Behavioral Health & Social Service Providers +103TA0400X,Psychologist Addiction (Substance Use Disorder),Behavioral Health & Social Service Providers +103TA0700X,Psychologist Adult Development & Aging,Behavioral Health & Social Service Providers +103TB0200X,Psychologist Cognitive & Behavioral,Behavioral Health & Social Service Providers +103TC0700X,Psychologist Clinical,Behavioral Health & Social Service Providers +103TC1900X,Psychologist Counseling,Behavioral Health & Social Service Providers +103TC2200X,Psychologist Clinical Child & Adolescent,Behavioral Health & Social Service Providers +103TE1000X,Psychologist Educational,Behavioral Health & Social Service Providers +103TE1100X,Psychologist Exercise & Sports,Behavioral Health & Social Service Providers +103TF0000X,Psychologist Family,Behavioral Health & Social Service Providers +103TF0200X,Psychologist Forensic,Behavioral Health & Social Service Providers +103TH0004X,Psychologist Health,Behavioral Health & Social Service Providers +103TH0100X,Psychologist Health Service,Behavioral Health & Social Service Providers +103TM1700X,Psychologist Men & Masculinity,Behavioral Health & Social Service Providers +103TM1800X,Psychologist Mental Retardation & Developmental Disabilities,Behavioral Health & Social Service Providers +103TP0016X,Psychologist Prescribing (Medical),Behavioral Health & Social Service Providers +103TP0814X,Psychologist Psychoanalysis,Behavioral Health & Social Service Providers +103TP2700X,Psychologist Psychotherapy,Behavioral Health & Social Service Providers +103TP2701X,Psychologist Group Psychotherapy,Behavioral Health & Social Service Providers +103TR0400X,Psychologist Rehabilitation,Behavioral Health & Social Service Providers +103TS0200X,Psychologist School,Behavioral Health & Social Service Providers +103TW0100X,Psychologist Women,Behavioral Health & Social Service Providers +104100000X,Social Worker ,Behavioral Health & Social Service Providers +1041C0700X,Social Worker Clinical,Behavioral Health & Social Service Providers +1041S0200X,Social Worker School,Behavioral Health & Social Service Providers +106E00000X,Assistant Behavior Analyst ,Behavioral Health & Social Service Providers +106H00000X,Marriage & Family Therapist ,Behavioral Health & Social Service Providers +106S00000X,Behavior Technician ,Behavioral Health & Social Service Providers +111N00000X,Chiropractor ,Chiropractic Providers +111NI0013X,Chiropractor Independent Medical Examiner,Chiropractic Providers +111NI0900X,Chiropractor Internist,Chiropractic Providers +111NN0400X,Chiropractor Neurology,Chiropractic Providers +111NN1001X,Chiropractor Nutrition,Chiropractic Providers +111NP0017X,Chiropractor Pediatric Chiropractor,Chiropractic Providers +111NR0200X,Chiropractor Radiology,Chiropractic Providers +111NR0400X,Chiropractor Rehabilitation,Chiropractic Providers +111NS0005X,Chiropractor Sports Physician,Chiropractic Providers +111NT0100X,Chiropractor Thermography,Chiropractic Providers +111NX0100X,Chiropractor Occupational Health,Chiropractic Providers +111NX0800X,Chiropractor Orthopedic,Chiropractic Providers +122300000X,Dentist ,Dental Providers +1223D0001X,Dentist Dental Public Health,Dental Providers +1223D0004X,Dentist Dentist Anesthesiologist,Dental Providers +1223E0200X,Dentist Endodontics,Dental Providers +1223G0001X,Dentist General Practice,Dental Providers +1223P0106X,Dentist Oral and Maxillofacial Pathology,Dental Providers +1223P0221X,Dentist Pediatric Dentistry,Dental Providers +1223P0300X,Dentist Periodontics,Dental Providers +1223P0700X,Dentist Prosthodontics,Dental Providers +1223S0112X,Dentist Oral and Maxillofacial Surgery,Dental Providers +1223X0008X,Dentist Oral and Maxillofacial Radiology,Dental Providers +1223X0400X,Dentist Orthodontics and Dentofacial Orthopedics,Dental Providers +122400000X,Denturist ,Dental Providers +124Q00000X,Dental Hygienist ,Dental Providers +125J00000X,Dental Therapist ,Dental Providers +125K00000X,Advanced Practice Dental Therapist ,Dental Providers +125Q00000X,Oral Medicinist ,Dental Providers +126800000X,Dental Assistant ,Dental Providers +126900000X,Dental Laboratory Technician ,Dental Providers +132700000X,Dietary Manager ,Dietary & Nutritional Service Providers +133N00000X,Nutritionist ,Dietary & Nutritional Service Providers +133NN1002X,"Nutritionist Nutrition, Education",Dietary & Nutritional Service Providers +133V00000X,"Dietitian, Registered ",Dietary & Nutritional Service Providers +133VN1004X,"Dietitian, Registered Nutrition, Pediatric",Dietary & Nutritional Service Providers +133VN1005X,"Dietitian, Registered Nutrition, Renal",Dietary & Nutritional Service Providers +133VN1006X,"Dietitian, Registered Nutrition, Metabolic",Dietary & Nutritional Service Providers +136A00000X,"Dietetic Technician, Registered ",Dietary & Nutritional Service Providers +146D00000X,Personal Emergency Response Attendant ,Emergency Medical Service Providers +146L00000X,"Emergency Medical Technician, Paramedic ",Emergency Medical Service Providers +146M00000X,"Emergency Medical Technician, Intermediate ",Emergency Medical Service Providers +146N00000X,"Emergency Medical Technician, Basic ",Emergency Medical Service Providers +152W00000X,Optometrist ,Eye and Vision Services Providers +152WC0802X,Optometrist Corneal and Contact Management,Eye and Vision Services Providers +152WL0500X,Optometrist Low Vision Rehabilitation,Eye and Vision Services Providers +152WP0200X,Optometrist Pediatrics,Eye and Vision Services Providers +152WS0006X,Optometrist Sports Vision,Eye and Vision Services Providers +152WV0400X,Optometrist Vision Therapy,Eye and Vision Services Providers +152WX0102X,Optometrist Occupational Vision,Eye and Vision Services Providers +156F00000X,Technician/Technologist ,Eye and Vision Services Providers +156FC0800X,Technician/Technologist Contact Lens,Eye and Vision Services Providers +156FC0801X,Technician/Technologist Contact Lens Fitter,Eye and Vision Services Providers +156FX1100X,Technician/Technologist Ophthalmic,Eye and Vision Services Providers +156FX1101X,Technician/Technologist Ophthalmic Assistant,Eye and Vision Services Providers +156FX1201X,Technician/Technologist Optometric Assistant,Eye and Vision Services Providers +156FX1202X,Technician/Technologist Optometric Technician,Eye and Vision Services Providers +156FX1700X,Technician/Technologist Ocularist,Eye and Vision Services Providers +156FX1800X,Technician/Technologist Optician,Eye and Vision Services Providers +156FX1900X,Technician/Technologist Orthoptist,Eye and Vision Services Providers +163W00000X,Registered Nurse ,Nursing Service Providers +163WA0400X,Registered Nurse Addiction (Substance Use Disorder),Nursing Service Providers +163WA2000X,Registered Nurse Administrator,Nursing Service Providers +163WC0200X,Registered Nurse Critical Care Medicine,Nursing Service Providers +163WC0400X,Registered Nurse Case Management,Nursing Service Providers +163WC1400X,Registered Nurse College Health,Nursing Service Providers +163WC1500X,Registered Nurse Community Health,Nursing Service Providers +163WC1600X,Registered Nurse Continuing Education/Staff Development,Nursing Service Providers +163WC2100X,Registered Nurse Continence Care,Nursing Service Providers +163WC3500X,Registered Nurse Cardiac Rehabilitation,Nursing Service Providers +163WD0400X,Registered Nurse Diabetes Educator,Nursing Service Providers +163WD1100X,"Registered Nurse Dialysis, Peritoneal",Nursing Service Providers +163WE0003X,Registered Nurse Emergency,Nursing Service Providers +163WE0900X,Registered Nurse Enterostomal Therapy,Nursing Service Providers +163WF0300X,Registered Nurse Flight,Nursing Service Providers +163WG0000X,Registered Nurse General Practice,Nursing Service Providers +163WG0100X,Registered Nurse Gastroenterology,Nursing Service Providers +163WG0600X,Registered Nurse Gerontology,Nursing Service Providers +163WH0200X,Registered Nurse Home Health,Nursing Service Providers +163WH0500X,Registered Nurse Hemodialysis,Nursing Service Providers +163WH1000X,Registered Nurse Hospice,Nursing Service Providers +163WI0500X,Registered Nurse Infusion Therapy,Nursing Service Providers +163WI0600X,Registered Nurse Infection Control,Nursing Service Providers +163WL0100X,Registered Nurse Lactation Consultant,Nursing Service Providers +163WM0102X,Registered Nurse Maternal Newborn,Nursing Service Providers +163WM0705X,Registered Nurse Medical-Surgical,Nursing Service Providers +163WM1400X,Registered Nurse Nurse Massage Therapist (NMT),Nursing Service Providers +163WN0002X,Registered Nurse Neonatal Intensive Care,Nursing Service Providers +163WN0003X,"Registered Nurse Neonatal, Low-Risk",Nursing Service Providers +163WN0300X,Registered Nurse Nephrology,Nursing Service Providers +163WN0800X,Registered Nurse Neuroscience,Nursing Service Providers +163WN1003X,Registered Nurse Nutrition Support,Nursing Service Providers +163WP0000X,Registered Nurse Pain Management,Nursing Service Providers +163WP0200X,Registered Nurse Pediatrics,Nursing Service Providers +163WP0218X,Registered Nurse Pediatric Oncology,Nursing Service Providers +163WP0807X,"Registered Nurse Psych/Mental Health, Child & Adolescent",Nursing Service Providers +163WP0808X,Registered Nurse Psych/Mental Health,Nursing Service Providers +163WP0809X,"Registered Nurse Psych/Mental Health, Adult",Nursing Service Providers +163WP1700X,Registered Nurse Perinatal,Nursing Service Providers +163WP2201X,Registered Nurse Ambulatory Care,Nursing Service Providers +163WR0006X,Registered Nurse Registered Nurse First Assistant,Nursing Service Providers +163WR0400X,Registered Nurse Rehabilitation,Nursing Service Providers +163WR1000X,Registered Nurse Reproductive Endocrinology/Infertility,Nursing Service Providers +163WS0121X,Registered Nurse Plastic Surgery,Nursing Service Providers +163WS0200X,Registered Nurse School,Nursing Service Providers +163WU0100X,Registered Nurse Urology,Nursing Service Providers +163WW0000X,Registered Nurse Wound Care,Nursing Service Providers +163WW0101X,"Registered Nurse Women's Health Care, Ambulatory",Nursing Service Providers +163WX0002X,"Registered Nurse Obstetric, High-Risk",Nursing Service Providers +163WX0003X,"Registered Nurse Obstetric, Inpatient",Nursing Service Providers +163WX0106X,Registered Nurse Occupational Health,Nursing Service Providers +163WX0200X,Registered Nurse Oncology,Nursing Service Providers +163WX0601X,Registered Nurse Otorhinolaryngology & Head-Neck,Nursing Service Providers +163WX0800X,Registered Nurse Orthopedic,Nursing Service Providers +163WX1100X,Registered Nurse Ophthalmic,Nursing Service Providers +163WX1500X,Registered Nurse Ostomy Care,Nursing Service Providers +164W00000X,Licensed Practical Nurse ,Nursing Service Providers +164X00000X,Licensed Vocational Nurse ,Nursing Service Providers +167G00000X,Licensed Psychiatric Technician ,Nursing Service Providers +170100000X,"Medical Genetics, Ph.D. Medical Genetics ",Other Service Providers +170300000X,"Genetic Counselor, MS ",Other Service Providers +171000000X,Military Health Care Provider ,Other Service Providers +1710I1002X,Military Health Care Provider Independent Duty Corpsman,Other Service Providers +1710I1003X,Military Health Care Provider Independent Duty Medical Technicians,Other Service Providers +171100000X,Acupuncturist ,Other Service Providers +171M00000X,Case Manager/Care Coordinator ,Other Service Providers +171R00000X,Interpreter ,Other Service Providers +171W00000X,Contractor ,Other Service Providers +171WH0202X,Contractor Home Modifications,Other Service Providers +171WV0202X,Contractor Vehicle Modifications,Other Service Providers +172A00000X,Driver ,Other Service Providers +172M00000X,Mechanotherapist ,Other Service Providers +172P00000X,Naprapath ,Other Service Providers +172V00000X,Community Health Worker ,Other Service Providers +173000000X,Legal Medicine ,Other Service Providers +173C00000X,Reflexologist ,Other Service Providers +173F00000X,"Sleep Specialist, PhD ",Other Service Providers +174200000X,Meals ,Other Service Providers +174400000X,Specialist ,Other Service Providers +1744G0900X,Specialist Graphics Designer,Other Service Providers +1744P3200X,Specialist Prosthetics Case Management,Other Service Providers +1744R1102X,Specialist Research Study,Other Service Providers +1744R1103X,Specialist Research Data Abstracter/Coder,Other Service Providers +174H00000X,Health Educator ,Other Service Providers +174M00000X,Veterinarian ,Other Service Providers +174MM1900X,Veterinarian Medical Research,Other Service Providers +174N00000X,"Lactation Consultant, Non-RN ",Other Service Providers +174V00000X,Clinical Ethicist ,Other Service Providers +175F00000X,Naturopath ,Other Service Providers +175L00000X,Homeopath ,Other Service Providers +175M00000X,"Midwife, Lay ",Other Service Providers +175T00000X,Peer Specialist ,Other Service Providers +176B00000X,Midwife ,Other Service Providers +176P00000X,Funeral Director ,Other Service Providers +177F00000X,Lodging ,Other Service Providers +183500000X,Pharmacist ,Pharmacy Service Providers +1835C0205X,Pharmacist Critical Care,Pharmacy Service Providers +1835G0000X,Pharmacist General Practice,Pharmacy Service Providers +1835G0303X,Pharmacist Geriatric,Pharmacy Service Providers +1835N0905X,Pharmacist Nuclear,Pharmacy Service Providers +1835N1003X,Pharmacist Nutrition Support,Pharmacy Service Providers +1835P0018X,Pharmacist Pharmacist Clinician (PhC)/ Clinical Pharmacy Specialist,Pharmacy Service Providers +1835P0200X,Pharmacist Pediatrics,Pharmacy Service Providers +1835P1200X,Pharmacist Pharmacotherapy,Pharmacy Service Providers +1835P1300X,Pharmacist Psychiatric,Pharmacy Service Providers +1835P2201X,Pharmacist Ambulatory Care,Pharmacy Service Providers +1835X0200X,Pharmacist Oncology,Pharmacy Service Providers +183700000X,Pharmacy Technician ,Pharmacy Service Providers +193200000X,Multi-Specialty ,Group +193400000X,Single Specialty ,Group +202C00000X,Independent Medical Examiner ,Allopathic & Osteopathic Physicians +202K00000X,Phlebology ,Allopathic & Osteopathic Physicians +204C00000X,"Neuromusculoskeletal Medicine, Sports Medicine ",Allopathic & Osteopathic Physicians +204D00000X,Neuromusculoskeletal Medicine & OMM ,Allopathic & Osteopathic Physicians +204E00000X,Oral & Maxillofacial Surgery ,Allopathic & Osteopathic Physicians +204F00000X,Transplant Surgery ,Allopathic & Osteopathic Physicians +204R00000X,Electrodiagnostic Medicine ,Allopathic & Osteopathic Physicians +207K00000X,Allergy & Immunology ,Allopathic & Osteopathic Physicians +207KA0200X,Allergy & Immunology Allergy,Allopathic & Osteopathic Physicians +207KI0005X,Allergy & Immunology Clinical & Laboratory Immunology,Allopathic & Osteopathic Physicians +207L00000X,Anesthesiology ,Allopathic & Osteopathic Physicians +207LA0401X,Anesthesiology Addiction Medicine,Allopathic & Osteopathic Physicians +207LC0200X,Anesthesiology Critical Care Medicine,Allopathic & Osteopathic Physicians +207LH0002X,Anesthesiology Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +207LP2900X,Anesthesiology Pain Medicine,Allopathic & Osteopathic Physicians +207LP3000X,Anesthesiology Pediatric Anesthesiology,Allopathic & Osteopathic Physicians +207N00000X,Dermatology ,Allopathic & Osteopathic Physicians +207ND0101X,Dermatology MOHS-Micrographic Surgery,Allopathic & Osteopathic Physicians +207ND0900X,Dermatology Dermatopathology,Allopathic & Osteopathic Physicians +207NI0002X,Dermatology Clinical & Laboratory Dermatological Immunology,Allopathic & Osteopathic Physicians +207NP0225X,Dermatology Pediatric Dermatology,Allopathic & Osteopathic Physicians +207NS0135X,Dermatology Procedural Dermatology,Allopathic & Osteopathic Physicians +207P00000X,Emergency Medicine ,Allopathic & Osteopathic Physicians +207PE0004X,Emergency Medicine Emergency Medical Services,Allopathic & Osteopathic Physicians +207PE0005X,Emergency Medicine Undersea and Hyperbaric Medicine,Allopathic & Osteopathic Physicians +207PH0002X,Emergency Medicine Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +207PP0204X,Emergency Medicine Pediatric Emergency Medicine,Allopathic & Osteopathic Physicians +207PS0010X,Emergency Medicine Sports Medicine,Allopathic & Osteopathic Physicians +207PT0002X,Emergency Medicine Medical Toxicology,Allopathic & Osteopathic Physicians +207Q00000X,Family Medicine ,Allopathic & Osteopathic Physicians +207QA0000X,Family Medicine Adolescent Medicine,Allopathic & Osteopathic Physicians +207QA0401X,Family Medicine Addiction Medicine,Allopathic & Osteopathic Physicians +207QA0505X,Family Medicine Adult Medicine,Allopathic & Osteopathic Physicians +207QB0002X,Family Medicine Obesity Medicine,Allopathic & Osteopathic Physicians +207QG0300X,Family Medicine Geriatric Medicine,Allopathic & Osteopathic Physicians +207QH0002X,Family Medicine Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +207QS0010X,Family Medicine Sports Medicine,Allopathic & Osteopathic Physicians +207QS1201X,Family Medicine Sleep Medicine,Allopathic & Osteopathic Physicians +207R00000X,Internal Medicine ,Allopathic & Osteopathic Physicians +207RA0000X,Internal Medicine Adolescent Medicine,Allopathic & Osteopathic Physicians +207RA0001X,Internal Medicine Advanced Heart Failure and Transplant Cardiology,Allopathic & Osteopathic Physicians +207RA0201X,Internal Medicine Allergy & Immunology,Allopathic & Osteopathic Physicians +207RA0401X,Internal Medicine Addiction Medicine,Allopathic & Osteopathic Physicians +207RB0002X,Internal Medicine Obesity Medicine,Allopathic & Osteopathic Physicians +207RC0000X,Internal Medicine Cardiovascular Disease,Allopathic & Osteopathic Physicians +207RC0001X,Internal Medicine Clinical Cardiac Electrophysiology,Allopathic & Osteopathic Physicians +207RC0200X,Internal Medicine Critical Care Medicine,Allopathic & Osteopathic Physicians +207RE0101X,"Internal Medicine Endocrinology, Diabetes & Metabolism",Allopathic & Osteopathic Physicians +207RG0100X,Internal Medicine Gastroenterology,Allopathic & Osteopathic Physicians +207RG0300X,Internal Medicine Geriatric Medicine,Allopathic & Osteopathic Physicians +207RH0000X,Internal Medicine Hematology,Allopathic & Osteopathic Physicians +207RH0002X,Internal Medicine Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +207RH0003X,Internal Medicine Hematology & Oncology,Allopathic & Osteopathic Physicians +207RH0005X,Internal Medicine Hypertension Specialist,Allopathic & Osteopathic Physicians +207RI0001X,Internal Medicine Clinical & Laboratory Immunology,Allopathic & Osteopathic Physicians +207RI0008X,Internal Medicine Hepatology,Allopathic & Osteopathic Physicians +207RI0011X,Internal Medicine Interventional Cardiology,Allopathic & Osteopathic Physicians +207RI0200X,Internal Medicine Infectious Disease,Allopathic & Osteopathic Physicians +207RM1200X,Internal Medicine Magnetic Resonance Imaging (MRI),Allopathic & Osteopathic Physicians +207RN0300X,Internal Medicine Nephrology,Allopathic & Osteopathic Physicians +207RP1001X,Internal Medicine Pulmonary Disease,Allopathic & Osteopathic Physicians +207RR0500X,Internal Medicine Rheumatology,Allopathic & Osteopathic Physicians +207RS0010X,Internal Medicine Sports Medicine,Allopathic & Osteopathic Physicians +207RS0012X,Internal Medicine Sleep Medicine,Allopathic & Osteopathic Physicians +207RT0003X,Internal Medicine Transplant Hepatology,Allopathic & Osteopathic Physicians +207RX0202X,Internal Medicine Medical Oncology,Allopathic & Osteopathic Physicians +207SC0300X,Medical Genetics Clinical Cytogenetic,Allopathic & Osteopathic Physicians +207SG0201X,Medical Genetics Clinical Genetics (M.D.),Allopathic & Osteopathic Physicians +207SG0202X,Medical Genetics Clinical Biochemical Genetics,Allopathic & Osteopathic Physicians +207SG0203X,Medical Genetics Clinical Molecular Genetics,Allopathic & Osteopathic Physicians +207SG0205X,Medical Genetics Ph.D. Medical Genetics,Allopathic & Osteopathic Physicians +207SM0001X,Medical Genetics Molecular Genetic Pathology,Allopathic & Osteopathic Physicians +207T00000X,Neurological Surgery ,Allopathic & Osteopathic Physicians +207U00000X,Nuclear Medicine ,Allopathic & Osteopathic Physicians +207UN0901X,Nuclear Medicine Nuclear Cardiology,Allopathic & Osteopathic Physicians +207UN0902X,Nuclear Medicine Nuclear Imaging & Therapy,Allopathic & Osteopathic Physicians +207UN0903X,Nuclear Medicine In Vivo & In Vitro Nuclear Medicine,Allopathic & Osteopathic Physicians +207V00000X,Obstetrics & Gynecology ,Allopathic & Osteopathic Physicians +207VB0002X,Obstetrics & Gynecology Obesity Medicine,Allopathic & Osteopathic Physicians +207VC0200X,Obstetrics & Gynecology Critical Care Medicine,Allopathic & Osteopathic Physicians +207VE0102X,Obstetrics & Gynecology Reproductive Endocrinology,Allopathic & Osteopathic Physicians +207VF0040X,Obstetrics & Gynecology Female Pelvic Medicine and Reconstructive Surgery,Allopathic & Osteopathic Physicians +207VG0400X,Obstetrics & Gynecology Gynecology,Allopathic & Osteopathic Physicians +207VH0002X,Obstetrics & Gynecology Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +207VM0101X,Obstetrics & Gynecology Maternal & Fetal Medicine,Allopathic & Osteopathic Physicians +207VX0000X,Obstetrics & Gynecology Obstetrics,Allopathic & Osteopathic Physicians +207VX0201X,Obstetrics & Gynecology Gynecologic Oncology,Allopathic & Osteopathic Physicians +207W00000X,Ophthalmology ,Allopathic & Osteopathic Physicians +207WX0009X,Ophthalmology Glaucoma Specialist,Allopathic & Osteopathic Physicians +207WX0107X,Ophthalmology Retina Specialist,Allopathic & Osteopathic Physicians +207WX0108X,Ophthalmology Uveitis and Ocular Inflammatory Disease,Allopathic & Osteopathic Physicians +207WX0109X,Ophthalmology Neuro-ophthalmology,Allopathic & Osteopathic Physicians +207WX0110X,Ophthalmology Pediatric Ophthalmology and Strabismus Specialist,Allopathic & Osteopathic Physicians +207WX0200X,Ophthalmology Ophthalmic Plastic and Reconstructive Surgery,Allopathic & Osteopathic Physicians +207X00000X,Orthopaedic Surgery ,Allopathic & Osteopathic Physicians +207XP3100X,Orthopaedic Surgery Pediatric Orthopaedic Surgery,Allopathic & Osteopathic Physicians +207XS0106X,Orthopaedic Surgery Hand Surgery,Allopathic & Osteopathic Physicians +207XS0114X,Orthopaedic Surgery Adult Reconstructive Orthopaedic Surgery,Allopathic & Osteopathic Physicians +207XS0117X,Orthopaedic Surgery Orthopaedic Surgery of the Spine,Allopathic & Osteopathic Physicians +207XX0004X,Orthopaedic Surgery Foot and Ankle Surgery,Allopathic & Osteopathic Physicians +207XX0005X,Orthopaedic Surgery Sports Medicine,Allopathic & Osteopathic Physicians +207XX0801X,Orthopaedic Surgery Orthopaedic Trauma,Allopathic & Osteopathic Physicians +207Y00000X,Otolaryngology ,Allopathic & Osteopathic Physicians +207YP0228X,Otolaryngology Pediatric Otolaryngology,Allopathic & Osteopathic Physicians +207YS0012X,Otolaryngology Sleep Medicine,Allopathic & Osteopathic Physicians +207YS0123X,Otolaryngology Facial Plastic Surgery,Allopathic & Osteopathic Physicians +207YX0007X,Otolaryngology Plastic Surgery within the Head & Neck,Allopathic & Osteopathic Physicians +207YX0602X,Otolaryngology Otolaryngic Allergy,Allopathic & Osteopathic Physicians +207YX0901X,Otolaryngology Otology & Neurotology,Allopathic & Osteopathic Physicians +207YX0905X,Otolaryngology Otolaryngology/Facial Plastic Surgery,Allopathic & Osteopathic Physicians +207ZB0001X,Pathology Blood Banking & Transfusion Medicine,Allopathic & Osteopathic Physicians +207ZC0006X,Pathology Clinical Pathology,Allopathic & Osteopathic Physicians +207ZC0008X,Pathology Clinical Informatics,Allopathic & Osteopathic Physicians +207ZC0500X,Pathology Cytopathology,Allopathic & Osteopathic Physicians +207ZD0900X,Pathology Dermatopathology,Allopathic & Osteopathic Physicians +207ZF0201X,Pathology Forensic Pathology,Allopathic & Osteopathic Physicians +207ZH0000X,Pathology Hematology,Allopathic & Osteopathic Physicians +207ZI0100X,Pathology Immunopathology,Allopathic & Osteopathic Physicians +207ZM0300X,Pathology Medical Microbiology,Allopathic & Osteopathic Physicians +207ZN0500X,Pathology Neuropathology,Allopathic & Osteopathic Physicians +207ZP0007X,Pathology Molecular Genetic Pathology,Allopathic & Osteopathic Physicians +207ZP0101X,Pathology Anatomic Pathology,Allopathic & Osteopathic Physicians +207ZP0102X,Pathology Anatomic Pathology & Clinical Pathology,Allopathic & Osteopathic Physicians +207ZP0104X,Pathology Chemical Pathology,Allopathic & Osteopathic Physicians +207ZP0105X,Pathology Clinical Pathology/Laboratory Medicine,Allopathic & Osteopathic Physicians +207ZP0213X,Pathology Pediatric Pathology,Allopathic & Osteopathic Physicians +208000000X,Pediatrics ,Allopathic & Osteopathic Physicians +2080A0000X,Pediatrics Adolescent Medicine,Allopathic & Osteopathic Physicians +2080B0002X,Pediatrics Obesity Medicine,Allopathic & Osteopathic Physicians +2080C0008X,Pediatrics Child Abuse Pediatrics,Allopathic & Osteopathic Physicians +2080H0002X,Pediatrics Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +2080I0007X,Pediatrics Clinical & Laboratory Immunology,Allopathic & Osteopathic Physicians +2080N0001X,Pediatrics Neonatal-Perinatal Medicine,Allopathic & Osteopathic Physicians +2080P0006X,Pediatrics Developmental - Behavioral Pediatrics,Allopathic & Osteopathic Physicians +2080P0008X,Pediatrics Neurodevelopmental Disabilities,Allopathic & Osteopathic Physicians +2080P0201X,Pediatrics Pediatric Allergy/Immunology,Allopathic & Osteopathic Physicians +2080P0202X,Pediatrics Pediatric Cardiology,Allopathic & Osteopathic Physicians +2080P0203X,Pediatrics Pediatric Critical Care Medicine,Allopathic & Osteopathic Physicians +2080P0204X,Pediatrics Pediatric Emergency Medicine,Allopathic & Osteopathic Physicians +2080P0205X,Pediatrics Pediatric Endocrinology,Allopathic & Osteopathic Physicians +2080P0206X,Pediatrics Pediatric Gastroenterology,Allopathic & Osteopathic Physicians +2080P0207X,Pediatrics Pediatric Hematology-Oncology,Allopathic & Osteopathic Physicians +2080P0208X,Pediatrics Pediatric Infectious Diseases,Allopathic & Osteopathic Physicians +2080P0210X,Pediatrics Pediatric Nephrology,Allopathic & Osteopathic Physicians +2080P0214X,Pediatrics Pediatric Pulmonology,Allopathic & Osteopathic Physicians +2080P0216X,Pediatrics Pediatric Rheumatology,Allopathic & Osteopathic Physicians +2080S0010X,Pediatrics Sports Medicine,Allopathic & Osteopathic Physicians +2080S0012X,Pediatrics Sleep Medicine,Allopathic & Osteopathic Physicians +2080T0002X,Pediatrics Medical Toxicology,Allopathic & Osteopathic Physicians +2080T0004X,Pediatrics Pediatric Transplant Hepatology,Allopathic & Osteopathic Physicians +208100000X,Physical Medicine & Rehabilitation ,Allopathic & Osteopathic Physicians +2081H0002X,Physical Medicine & Rehabilitation Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +2081N0008X,Physical Medicine & Rehabilitation Neuromuscular Medicine,Allopathic & Osteopathic Physicians +2081P0004X,Physical Medicine & Rehabilitation Spinal Cord Injury Medicine,Allopathic & Osteopathic Physicians +2081P0010X,Physical Medicine & Rehabilitation Pediatric Rehabilitation Medicine,Allopathic & Osteopathic Physicians +2081P0301X,Physical Medicine & Rehabilitation Brain Injury Medicine,Allopathic & Osteopathic Physicians +2081P2900X,Physical Medicine & Rehabilitation Pain Medicine,Allopathic & Osteopathic Physicians +2081S0010X,Physical Medicine & Rehabilitation Sports Medicine,Allopathic & Osteopathic Physicians +208200000X,Plastic Surgery ,Allopathic & Osteopathic Physicians +2082S0099X,Plastic Surgery Plastic Surgery Within the Head and Neck,Allopathic & Osteopathic Physicians +2082S0105X,Plastic Surgery Surgery of the Hand,Allopathic & Osteopathic Physicians +2083A0100X,Preventive Medicine Aerospace Medicine,Allopathic & Osteopathic Physicians +2083B0002X,Preventive Medicine Obesity Medicine,Allopathic & Osteopathic Physicians +2083C0008X,Preventive Medicine Clinical Informatics,Allopathic & Osteopathic Physicians +2083P0011X,Preventive Medicine Undersea and Hyperbaric Medicine,Allopathic & Osteopathic Physicians +2083P0500X,Preventive Medicine Preventive Medicine/Occupational Environmental Medicine,Allopathic & Osteopathic Physicians +2083P0901X,Preventive Medicine Public Health & General Preventive Medicine,Allopathic & Osteopathic Physicians +2083S0010X,Preventive Medicine Sports Medicine,Allopathic & Osteopathic Physicians +2083T0002X,Preventive Medicine Medical Toxicology,Allopathic & Osteopathic Physicians +2083X0100X,Preventive Medicine Occupational Medicine,Allopathic & Osteopathic Physicians +2084A0401X,Psychiatry & Neurology Addiction Medicine,Allopathic & Osteopathic Physicians +2084A2900X,Psychiatry & Neurology Neurocritical Care,Allopathic & Osteopathic Physicians +2084B0002X,Psychiatry & Neurology Obesity Medicine,Allopathic & Osteopathic Physicians +2084B0040X,Psychiatry & Neurology Behavioral Neurology & Neuropsychiatry,Allopathic & Osteopathic Physicians +2084D0003X,Psychiatry & Neurology Diagnostic Neuroimaging,Allopathic & Osteopathic Physicians +2084F0202X,Psychiatry & Neurology Forensic Psychiatry,Allopathic & Osteopathic Physicians +2084H0002X,Psychiatry & Neurology Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +2084N0008X,Psychiatry & Neurology Neuromuscular Medicine,Allopathic & Osteopathic Physicians +2084N0400X,Psychiatry & Neurology Neurology,Allopathic & Osteopathic Physicians +2084N0402X,Psychiatry & Neurology Neurology with Special Qualifications in Child Neurology,Allopathic & Osteopathic Physicians +2084N0600X,Psychiatry & Neurology Clinical Neurophysiology,Allopathic & Osteopathic Physicians +2084P0005X,Psychiatry & Neurology Neurodevelopmental Disabilities,Allopathic & Osteopathic Physicians +2084P0015X,Psychiatry & Neurology Psychosomatic Medicine,Allopathic & Osteopathic Physicians +2084P0301X,Psychiatry & Neurology Brain Injury Medicine,Allopathic & Osteopathic Physicians +2084P0800X,Psychiatry & Neurology Psychiatry,Allopathic & Osteopathic Physicians +2084P0802X,Psychiatry & Neurology Addiction Psychiatry,Allopathic & Osteopathic Physicians +2084P0804X,Psychiatry & Neurology Child & Adolescent Psychiatry,Allopathic & Osteopathic Physicians +2084P0805X,Psychiatry & Neurology Geriatric Psychiatry,Allopathic & Osteopathic Physicians +2084P2900X,Psychiatry & Neurology Pain Medicine,Allopathic & Osteopathic Physicians +2084S0010X,Psychiatry & Neurology Sports Medicine,Allopathic & Osteopathic Physicians +2084S0012X,Psychiatry & Neurology Sleep Medicine,Allopathic & Osteopathic Physicians +2084V0102X,Psychiatry & Neurology Vascular Neurology,Allopathic & Osteopathic Physicians +2085B0100X,Radiology Body Imaging,Allopathic & Osteopathic Physicians +2085D0003X,Radiology Diagnostic Neuroimaging,Allopathic & Osteopathic Physicians +2085H0002X,Radiology Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +2085N0700X,Radiology Neuroradiology,Allopathic & Osteopathic Physicians +2085N0904X,Radiology Nuclear Radiology,Allopathic & Osteopathic Physicians +2085P0229X,Radiology Pediatric Radiology,Allopathic & Osteopathic Physicians +2085R0001X,Radiology Radiation Oncology,Allopathic & Osteopathic Physicians +2085R0202X,Radiology Diagnostic Radiology,Allopathic & Osteopathic Physicians +2085R0203X,Radiology Therapeutic Radiology,Allopathic & Osteopathic Physicians +2085R0204X,Radiology Vascular & Interventional Radiology,Allopathic & Osteopathic Physicians +2085R0205X,Radiology Radiological Physics,Allopathic & Osteopathic Physicians +2085U0001X,Radiology Diagnostic Ultrasound,Allopathic & Osteopathic Physicians +208600000X,Surgery ,Allopathic & Osteopathic Physicians +2086H0002X,Surgery Hospice and Palliative Medicine,Allopathic & Osteopathic Physicians +2086S0102X,Surgery Surgical Critical Care,Allopathic & Osteopathic Physicians +2086S0105X,Surgery Surgery of the Hand,Allopathic & Osteopathic Physicians +2086S0120X,Surgery Pediatric Surgery,Allopathic & Osteopathic Physicians +2086S0122X,Surgery Plastic and Reconstructive Surgery,Allopathic & Osteopathic Physicians +2086S0127X,Surgery Trauma Surgery,Allopathic & Osteopathic Physicians +2086S0129X,Surgery Vascular Surgery,Allopathic & Osteopathic Physicians +2086X0206X,Surgery Surgical Oncology,Allopathic & Osteopathic Physicians +208800000X,Urology ,Allopathic & Osteopathic Physicians +2088F0040X,Urology Female Pelvic Medicine and Reconstructive Surgery,Allopathic & Osteopathic Physicians +2088P0231X,Urology Pediatric Urology,Allopathic & Osteopathic Physicians +208C00000X,Colon & Rectal Surgery ,Allopathic & Osteopathic Physicians +208D00000X,General Practice ,Allopathic & Osteopathic Physicians +208G00000X,Thoracic Surgery (Cardiothoracic Vascular Surgery) ,Allopathic & Osteopathic Physicians +208M00000X,Hospitalist ,Allopathic & Osteopathic Physicians +208U00000X,Clinical Pharmacology ,Allopathic & Osteopathic Physicians +208VP0000X,Pain Medicine Pain Medicine,Allopathic & Osteopathic Physicians +208VP0014X,Pain Medicine Interventional Pain Medicine,Allopathic & Osteopathic Physicians +209800000X,Legal Medicine ,Allopathic & Osteopathic Physicians +211D00000X,"Assistant, Podiatric ",Podiatric Medicine & Surgery Service Providers +213E00000X,Podiatrist ,Podiatric Medicine & Surgery Service Providers +213EG0000X,Podiatrist General Practice,Podiatric Medicine & Surgery Service Providers +213EP0504X,Podiatrist Public Medicine,Podiatric Medicine & Surgery Service Providers +213EP1101X,Podiatrist Primary Podiatric Medicine,Podiatric Medicine & Surgery Service Providers +213ER0200X,Podiatrist Radiology,Podiatric Medicine & Surgery Service Providers +213ES0000X,Podiatrist Sports Medicine,Podiatric Medicine & Surgery Service Providers +213ES0103X,Podiatrist Foot & Ankle Surgery,Podiatric Medicine & Surgery Service Providers +213ES0131X,Podiatrist Foot Surgery,Podiatric Medicine & Surgery Service Providers +221700000X,Art Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +222Q00000X,Developmental Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +222Z00000X,Orthotist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224900000X,Mastectomy Fitter ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224L00000X,Pedorthist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224P00000X,Prosthetist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224Y00000X,Clinical Exercise Physiologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224Z00000X,Occupational Therapy Assistant ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224ZE0001X,Occupational Therapy Assistant Environmental Modification,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224ZF0002X,"Occupational Therapy Assistant Feeding, Eating & Swallowing","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224ZL0004X,Occupational Therapy Assistant Low Vision,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +224ZR0403X,Occupational Therapy Assistant Driving and Community Mobility,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225000000X,Orthotic Fitter ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225100000X,Physical Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251C2600X,Physical Therapist Cardiopulmonary,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251E1200X,Physical Therapist Ergonomics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251E1300X,"Physical Therapist Electrophysiology, Clinical","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251G0304X,Physical Therapist Geriatrics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251H1200X,Physical Therapist Hand,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251H1300X,Physical Therapist Human Factors,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251N0400X,Physical Therapist Neurology,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251P0200X,Physical Therapist Pediatrics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251S0007X,Physical Therapist Sports,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2251X0800X,Physical Therapist Orthopedic,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225200000X,Physical Therapy Assistant ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225400000X,Rehabilitation Practitioner ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225500000X,Specialist/Technologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2255A2300X,Specialist/Technologist Athletic Trainer,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2255R0406X,"Specialist/Technologist Rehabilitation, Blind","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225600000X,Dance Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225700000X,Massage Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225800000X,Recreation Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225A00000X,Music Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225B00000X,Pulmonary Function Technologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225C00000X,Rehabilitation Counselor ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225CA2400X,Rehabilitation Counselor Assistive Technology Practitioner,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225CA2500X,Rehabilitation Counselor Assistive Technology Supplier,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225CX0006X,Rehabilitation Counselor Orientation and Mobility Training Provider,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225X00000X,Occupational Therapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XE0001X,Occupational Therapist Environmental Modification,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XE1200X,Occupational Therapist Ergonomics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XF0002X,"Occupational Therapist Feeding, Eating & Swallowing","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XG0600X,Occupational Therapist Gerontology,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XH1200X,Occupational Therapist Hand,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XH1300X,Occupational Therapist Human Factors,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XL0004X,Occupational Therapist Low Vision,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XM0800X,Occupational Therapist Mental Health,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XN1300X,Occupational Therapist Neurorehabilitation,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XP0019X,Occupational Therapist Physical Rehabilitation,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XP0200X,Occupational Therapist Pediatrics,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +225XR0403X,Occupational Therapist Driving and Community Mobility,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +226000000X,Recreational Therapist Assistant ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +226300000X,Kinesiotherapist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +227800000X,"Respiratory Therapist, Certified ","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278C0205X,"Respiratory Therapist, Certified Critical Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278E0002X,"Respiratory Therapist, Certified Emergency Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278E1000X,"Respiratory Therapist, Certified Educational","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278G0305X,"Respiratory Therapist, Certified Geriatric Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278G1100X,"Respiratory Therapist, Certified General Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278H0200X,"Respiratory Therapist, Certified Home Health","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P1004X,"Respiratory Therapist, Certified Pulmonary Diagnostics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P1005X,"Respiratory Therapist, Certified Pulmonary Rehabilitation","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P1006X,"Respiratory Therapist, Certified Pulmonary Function Technologist","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P3800X,"Respiratory Therapist, Certified Palliative/Hospice","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P3900X,"Respiratory Therapist, Certified Neonatal/Pediatrics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278P4000X,"Respiratory Therapist, Certified Patient Transport","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2278S1500X,"Respiratory Therapist, Certified SNF/Subacute Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +227900000X,"Respiratory Therapist, Registered ","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279C0205X,"Respiratory Therapist, Registered Critical Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279E0002X,"Respiratory Therapist, Registered Emergency Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279E1000X,"Respiratory Therapist, Registered Educational","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279G0305X,"Respiratory Therapist, Registered Geriatric Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279G1100X,"Respiratory Therapist, Registered General Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279H0200X,"Respiratory Therapist, Registered Home Health","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P1004X,"Respiratory Therapist, Registered Pulmonary Diagnostics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P1005X,"Respiratory Therapist, Registered Pulmonary Rehabilitation","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P1006X,"Respiratory Therapist, Registered Pulmonary Function Technologist","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P3800X,"Respiratory Therapist, Registered Palliative/Hospice","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P3900X,"Respiratory Therapist, Registered Neonatal/Pediatrics","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279P4000X,"Respiratory Therapist, Registered Patient Transport","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +2279S1500X,"Respiratory Therapist, Registered SNF/Subacute Care","Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +229N00000X,Anaplastologist ,"Respiratory, Developmental, Rehabilitative and Restorative Service Providers" +231H00000X,Audiologist ,"Speech, Language and Hearing Service Providers" +231HA2400X,Audiologist Assistive Technology Practitioner,"Speech, Language and Hearing Service Providers" +231HA2500X,Audiologist Assistive Technology Supplier,"Speech, Language and Hearing Service Providers" +235500000X,Specialist/Technologist ,"Speech, Language and Hearing Service Providers" +2355A2700X,Specialist/Technologist Audiology Assistant,"Speech, Language and Hearing Service Providers" +2355S0801X,Specialist/Technologist Speech-Language Assistant,"Speech, Language and Hearing Service Providers" +235Z00000X,Speech-Language Pathologist ,"Speech, Language and Hearing Service Providers" +237600000X,Audiologist-Hearing Aid Fitter ,"Speech, Language and Hearing Service Providers" +237700000X,Hearing Instrument Specialist ,"Speech, Language and Hearing Service Providers" +242T00000X,Perfusionist ,"Technologists, Technicians & Other Technical Service Providers" +243U00000X,Radiology Practitioner Assistant ,"Technologists, Technicians & Other Technical Service Providers" +246Q00000X,"Spec/Tech, Pathology ","Technologists, Technicians & Other Technical Service Providers" +246QB0000X,"Spec/Tech, Pathology Blood Banking","Technologists, Technicians & Other Technical Service Providers" +246QC1000X,"Spec/Tech, Pathology Chemistry","Technologists, Technicians & Other Technical Service Providers" +246QC2700X,"Spec/Tech, Pathology Cytotechnology","Technologists, Technicians & Other Technical Service Providers" +246QH0000X,"Spec/Tech, Pathology Hematology","Technologists, Technicians & Other Technical Service Providers" +246QH0401X,"Spec/Tech, Pathology Hemapheresis Practitioner","Technologists, Technicians & Other Technical Service Providers" +246QH0600X,"Spec/Tech, Pathology Histology","Technologists, Technicians & Other Technical Service Providers" +246QI0000X,"Spec/Tech, Pathology Immunology","Technologists, Technicians & Other Technical Service Providers" +246QL0900X,"Spec/Tech, Pathology Laboratory Management","Technologists, Technicians & Other Technical Service Providers" +246QL0901X,"Spec/Tech, Pathology Laboratory Management, Diplomate","Technologists, Technicians & Other Technical Service Providers" +246QM0706X,"Spec/Tech, Pathology Medical Technologist","Technologists, Technicians & Other Technical Service Providers" +246QM0900X,"Spec/Tech, Pathology Microbiology","Technologists, Technicians & Other Technical Service Providers" +246R00000X,"Technician, Pathology ","Technologists, Technicians & Other Technical Service Providers" +246RH0600X,"Technician, Pathology Histology","Technologists, Technicians & Other Technical Service Providers" +246RM2200X,"Technician, Pathology Medical Laboratory","Technologists, Technicians & Other Technical Service Providers" +246RP1900X,"Technician, Pathology Phlebotomy","Technologists, Technicians & Other Technical Service Providers" +246W00000X,"Technician, Cardiology ","Technologists, Technicians & Other Technical Service Providers" +246X00000X,"Spec/Tech, Cardiovascular ","Technologists, Technicians & Other Technical Service Providers" +246XC2901X,"Spec/Tech, Cardiovascular Cardiovascular Invasive Specialist","Technologists, Technicians & Other Technical Service Providers" +246XC2903X,"Spec/Tech, Cardiovascular Vascular Specialist","Technologists, Technicians & Other Technical Service Providers" +246XS1301X,"Spec/Tech, Cardiovascular Sonography","Technologists, Technicians & Other Technical Service Providers" +246Y00000X,"Spec/Tech, Health Info ","Technologists, Technicians & Other Technical Service Providers" +246YC3301X,"Spec/Tech, Health Info Coding Specialist, Hospital Based","Technologists, Technicians & Other Technical Service Providers" +246YC3302X,"Spec/Tech, Health Info Coding Specialist, Physician Office Based","Technologists, Technicians & Other Technical Service Providers" +246YR1600X,"Spec/Tech, Health Info Registered Record Administrator","Technologists, Technicians & Other Technical Service Providers" +246Z00000X,"Specialist/Technologist, Other ","Technologists, Technicians & Other Technical Service Providers" +246ZA2600X,"Specialist/Technologist, Other Art, Medical","Technologists, Technicians & Other Technical Service Providers" +246ZB0301X,"Specialist/Technologist, Other Biomedical Engineering","Technologists, Technicians & Other Technical Service Providers" +246ZB0302X,"Specialist/Technologist, Other Biomedical Photographer","Technologists, Technicians & Other Technical Service Providers" +246ZB0500X,"Specialist/Technologist, Other Biochemist","Technologists, Technicians & Other Technical Service Providers" +246ZB0600X,"Specialist/Technologist, Other Biostatistician","Technologists, Technicians & Other Technical Service Providers" +246ZC0007X,"Specialist/Technologist, Other Surgical Assistant","Technologists, Technicians & Other Technical Service Providers" +246ZE0500X,"Specialist/Technologist, Other EEG","Technologists, Technicians & Other Technical Service Providers" +246ZE0600X,"Specialist/Technologist, Other Electroneurodiagnostic","Technologists, Technicians & Other Technical Service Providers" +246ZG0701X,"Specialist/Technologist, Other Graphics Methods","Technologists, Technicians & Other Technical Service Providers" +246ZG1000X,"Specialist/Technologist, Other Geneticist, Medical (PhD)","Technologists, Technicians & Other Technical Service Providers" +246ZI1000X,"Specialist/Technologist, Other Illustration, Medical","Technologists, Technicians & Other Technical Service Providers" +246ZN0300X,"Specialist/Technologist, Other Nephrology","Technologists, Technicians & Other Technical Service Providers" +246ZS0410X,"Specialist/Technologist, Other Surgical Technologist","Technologists, Technicians & Other Technical Service Providers" +246ZX2200X,"Specialist/Technologist, Other Orthopedic Assistant","Technologists, Technicians & Other Technical Service Providers" +247000000X,"Technician, Health Information ","Technologists, Technicians & Other Technical Service Providers" +2470A2800X,"Technician, Health Information Assistant Record Technician","Technologists, Technicians & Other Technical Service Providers" +247100000X,Radiologic Technologist ,"Technologists, Technicians & Other Technical Service Providers" +2471B0102X,Radiologic Technologist Bone Densitometry,"Technologists, Technicians & Other Technical Service Providers" +2471C1101X,Radiologic Technologist Cardiovascular-Interventional Technology,"Technologists, Technicians & Other Technical Service Providers" +2471C1106X,Radiologic Technologist Cardiac-Interventional Technology,"Technologists, Technicians & Other Technical Service Providers" +2471C3401X,Radiologic Technologist Computed Tomography,"Technologists, Technicians & Other Technical Service Providers" +2471C3402X,Radiologic Technologist Radiography,"Technologists, Technicians & Other Technical Service Providers" +2471M1202X,Radiologic Technologist Magnetic Resonance Imaging,"Technologists, Technicians & Other Technical Service Providers" +2471M2300X,Radiologic Technologist Mammography,"Technologists, Technicians & Other Technical Service Providers" +2471N0900X,Radiologic Technologist Nuclear Medicine Technology,"Technologists, Technicians & Other Technical Service Providers" +2471Q0001X,Radiologic Technologist Quality Management,"Technologists, Technicians & Other Technical Service Providers" +2471R0002X,Radiologic Technologist Radiation Therapy,"Technologists, Technicians & Other Technical Service Providers" +2471S1302X,Radiologic Technologist Sonography,"Technologists, Technicians & Other Technical Service Providers" +2471V0105X,Radiologic Technologist Vascular Sonography,"Technologists, Technicians & Other Technical Service Providers" +2471V0106X,Radiologic Technologist Vascular-Interventional Technology,"Technologists, Technicians & Other Technical Service Providers" +247200000X,"Technician, Other ","Technologists, Technicians & Other Technical Service Providers" +2472B0301X,"Technician, Other Biomedical Engineering","Technologists, Technicians & Other Technical Service Providers" +2472D0500X,"Technician, Other Darkroom","Technologists, Technicians & Other Technical Service Providers" +2472E0500X,"Technician, Other EEG","Technologists, Technicians & Other Technical Service Providers" +2472R0900X,"Technician, Other Renal Dialysis","Technologists, Technicians & Other Technical Service Providers" +2472V0600X,"Technician, Other Veterinary","Technologists, Technicians & Other Technical Service Providers" +247ZC0005X,"Pathology Clinical Laboratory Director, Non-physician","Technologists, Technicians & Other Technical Service Providers" +251300000X,Local Education Agency (LEA) ,Agencies +251B00000X,Case Management ,Agencies +251C00000X,"Day Training, Developmentally Disabled Services ",Agencies +251E00000X,Home Health ,Agencies +251F00000X,Home Infusion ,Agencies +251G00000X,"Hospice Care, Community Based ",Agencies +251J00000X,Nursing Care ,Agencies +251K00000X,Public Health or Welfare ,Agencies +251S00000X,Community/Behavioral Health ,Agencies +251T00000X,PACE Provider Organization ,Agencies +251V00000X,Voluntary or Charitable ,Agencies +251X00000X,Supports Brokerage ,Agencies +252Y00000X,Early Intervention Provider Agency ,Agencies +253J00000X,Foster Care Agency ,Agencies +253Z00000X,In Home Supportive Care ,Agencies +261Q00000X,Clinic/Center ,Ambulatory Health Care Facilities +261QA0005X,Clinic/Center Ambulatory Family Planning Facility,Ambulatory Health Care Facilities +261QA0006X,Clinic/Center Ambulatory Fertility Facility,Ambulatory Health Care Facilities +261QA0600X,Clinic/Center Adult Day Care,Ambulatory Health Care Facilities +261QA0900X,Clinic/Center Amputee,Ambulatory Health Care Facilities +261QA1903X,Clinic/Center Ambulatory Surgical,Ambulatory Health Care Facilities +261QA3000X,Clinic/Center Augmentative Communication,Ambulatory Health Care Facilities +261QB0400X,Clinic/Center Birthing,Ambulatory Health Care Facilities +261QC0050X,Clinic/Center Critical Access Hospital,Ambulatory Health Care Facilities +261QC1500X,Clinic/Center Community Health,Ambulatory Health Care Facilities +261QC1800X,Clinic/Center Corporate Health,Ambulatory Health Care Facilities +261QD0000X,Clinic/Center Dental,Ambulatory Health Care Facilities +261QD1600X,Clinic/Center Developmental Disabilities,Ambulatory Health Care Facilities +261QE0002X,Clinic/Center Emergency Care,Ambulatory Health Care Facilities +261QE0700X,Clinic/Center End-Stage Renal Disease (ESRD) Treatment,Ambulatory Health Care Facilities +261QE0800X,Clinic/Center Endoscopy,Ambulatory Health Care Facilities +261QF0050X,"Clinic/Center Family Planning, Non-Surgical",Ambulatory Health Care Facilities +261QF0400X,Clinic/Center Federally Qualified Health Center (FQHC),Ambulatory Health Care Facilities +261QG0250X,Clinic/Center Genetics,Ambulatory Health Care Facilities +261QH0100X,Clinic/Center Health Service,Ambulatory Health Care Facilities +261QH0700X,Clinic/Center Hearing and Speech,Ambulatory Health Care Facilities +261QI0500X,Clinic/Center Infusion Therapy,Ambulatory Health Care Facilities +261QL0400X,Clinic/Center Lithotripsy,Ambulatory Health Care Facilities +261QM0801X,Clinic/Center Mental Health (Including Community Mental Health Center),Ambulatory Health Care Facilities +261QM0850X,Clinic/Center Adult Mental Health,Ambulatory Health Care Facilities +261QM0855X,Clinic/Center Adolescent and Children Mental Health,Ambulatory Health Care Facilities +261QM1000X,Clinic/Center Migrant Health,Ambulatory Health Care Facilities +261QM1100X,Clinic/Center Military/U.S. Coast Guard Outpatient,Ambulatory Health Care Facilities +261QM1101X,Clinic/Center Military and U.S. Coast Guard Ambulatory Procedure,Ambulatory Health Care Facilities +261QM1102X,Clinic/Center Military Outpatient Operational (Transportable) Component,Ambulatory Health Care Facilities +261QM1103X,Clinic/Center Military Ambulatory Procedure Visits Operational (Transportable),Ambulatory Health Care Facilities +261QM1200X,Clinic/Center Magnetic Resonance Imaging (MRI),Ambulatory Health Care Facilities +261QM1300X,Clinic/Center Multi-Specialty,Ambulatory Health Care Facilities +261QM2500X,Clinic/Center Medical Specialty,Ambulatory Health Care Facilities +261QM2800X,Clinic/Center Methadone Clinic,Ambulatory Health Care Facilities +261QM3000X,Clinic/Center Medically Fragile Intants and Children Day Care,Ambulatory Health Care Facilities +261QP0904X,"Clinic/Center Public Health, Federal",Ambulatory Health Care Facilities +261QP0905X,"Clinic/Center Public Health, State or Local",Ambulatory Health Care Facilities +261QP1100X,Clinic/Center Podiatric,Ambulatory Health Care Facilities +261QP2000X,Clinic/Center Physical Therapy,Ambulatory Health Care Facilities +261QP2300X,Clinic/Center Primary Care,Ambulatory Health Care Facilities +261QP2400X,Clinic/Center Prison Health,Ambulatory Health Care Facilities +261QP3300X,Clinic/Center Pain,Ambulatory Health Care Facilities +261QR0200X,Clinic/Center Radiology,Ambulatory Health Care Facilities +261QR0206X,"Clinic/Center Radiology, Mammography",Ambulatory Health Care Facilities +261QR0207X,"Clinic/Center Radiology, Mobile Mammography",Ambulatory Health Care Facilities +261QR0208X,"Clinic/Center Radiology, Mobile",Ambulatory Health Care Facilities +261QR0400X,Clinic/Center Rehabilitation,Ambulatory Health Care Facilities +261QR0401X,"Clinic/Center Rehabilitation, Comprehensive Outpatient Rehabilitation Facility (CORF)",Ambulatory Health Care Facilities +261QR0404X,"Clinic/Center Rehabilitation, Cardiac Facilities",Ambulatory Health Care Facilities +261QR0405X,"Clinic/Center Rehabilitation, Substance Use Disorder",Ambulatory Health Care Facilities +261QR0800X,Clinic/Center Recovery Care,Ambulatory Health Care Facilities +261QR1100X,Clinic/Center Research,Ambulatory Health Care Facilities +261QR1300X,Clinic/Center Rural Health,Ambulatory Health Care Facilities +261QS0112X,Clinic/Center Oral and Maxillofacial Surgery,Ambulatory Health Care Facilities +261QS0132X,Clinic/Center Ophthalmologic Surgery,Ambulatory Health Care Facilities +261QS1000X,Clinic/Center Student Health,Ambulatory Health Care Facilities +261QS1200X,Clinic/Center Sleep Disorder Diagnostic,Ambulatory Health Care Facilities +261QU0200X,Clinic/Center Urgent Care,Ambulatory Health Care Facilities +261QV0200X,Clinic/Center VA,Ambulatory Health Care Facilities +261QX0100X,Clinic/Center Occupational Medicine,Ambulatory Health Care Facilities +261QX0200X,Clinic/Center Oncology,Ambulatory Health Care Facilities +261QX0203X,"Clinic/Center Oncology, Radiation",Ambulatory Health Care Facilities +273100000X,Epilepsy Unit ,Hospital Units +273R00000X,Psychiatric Unit ,Hospital Units +273Y00000X,Rehabilitation Unit ,Hospital Units +275N00000X,Medicare Defined Swing Bed Unit ,Hospital Units +276400000X,"Rehabilitation, Substance Use Disorder Unit ",Hospital Units +281P00000X,Chronic Disease Hospital ,Hospitals +281PC2000X,Chronic Disease Hospital Children,Hospitals +282E00000X,Long Term Care Hospital ,Hospitals +282J00000X,Religious Nonmedical Health Care Institution ,Hospitals +282N00000X,General Acute Care Hospital ,Hospitals +282NC0060X,General Acute Care Hospital Critical Access,Hospitals +282NC2000X,General Acute Care Hospital Children,Hospitals +282NR1301X,General Acute Care Hospital Rural,Hospitals +282NW0100X,General Acute Care Hospital Women,Hospitals +283Q00000X,Psychiatric Hospital ,Hospitals +283X00000X,Rehabilitation Hospital ,Hospitals +283XC2000X,Rehabilitation Hospital Children,Hospitals +284300000X,Special Hospital ,Hospitals +286500000X,Military Hospital ,Hospitals +2865C1500X,Military Hospital Community Health,Hospitals +2865M2000X,Military Hospital Military General Acute Care Hospital,Hospitals +2865X1600X,Military Hospital Military General Acute Care Hospital. Operational (Transportable),Hospitals +287300000X,Christian Science Sanitorium ,Hospitals +291900000X,Military Clinical Medical Laboratory ,Laboratories +291U00000X,Clinical Medical Laboratory ,Laboratories +292200000X,Dental Laboratory ,Laboratories +293D00000X,Physiological Laboratory ,Laboratories +302F00000X,Exclusive Provider Organization ,Managed Care Organizations +302R00000X,Health Maintenance Organization ,Managed Care Organizations +305R00000X,Preferred Provider Organization ,Managed Care Organizations +305S00000X,Point of Service ,Managed Care Organizations +310400000X,Assisted Living Facility ,Nursing & Custodial Care Facilities +3104A0625X,"Assisted Living Facility Assisted Living, Mental Illness",Nursing & Custodial Care Facilities +3104A0630X,"Assisted Living Facility Assisted Living, Behavioral Disturbances",Nursing & Custodial Care Facilities +310500000X,"Intermediate Care Facility, Mental Illness ",Nursing & Custodial Care Facilities +311500000X,Alzheimer Center (Dementia Center) ,Nursing & Custodial Care Facilities +311Z00000X,Custodial Care Facility ,Nursing & Custodial Care Facilities +311ZA0620X,Custodial Care Facility Adult Care Home,Nursing & Custodial Care Facilities +313M00000X,Nursing Facility/Intermediate Care Facility ,Nursing & Custodial Care Facilities +314000000X,Skilled Nursing Facility ,Nursing & Custodial Care Facilities +3140N1450X,"Skilled Nursing Facility Nursing Care, Pediatric",Nursing & Custodial Care Facilities +315D00000X,"Hospice, Inpatient ",Nursing & Custodial Care Facilities +315P00000X,"Intermediate Care Facility, Mentally Retarded ",Nursing & Custodial Care Facilities +317400000X,Christian Science Facility ,Nursing & Custodial Care Facilities +320600000X,"Residential Treatment Facility, Mental Retardation and/or Developmental Disabilities ",Residential Treatment Facilities +320700000X,"Residential Treatment Facility, Physical Disabilities ",Residential Treatment Facilities +320800000X,"Community Based Residential Treatment Facility, Mental Illness ",Residential Treatment Facilities +320900000X,"Community Based Residential Treatment, Mental Retardation and/or Developmental Disabilities ",Residential Treatment Facilities +322D00000X,"Residential Treatment Facility, Emotionally Disturbed Children ",Residential Treatment Facilities +323P00000X,Psychiatric Residential Treatment Facility ,Residential Treatment Facilities +324500000X,Substance Abuse Rehabilitation Facility ,Residential Treatment Facilities +3245S0500X,"Substance Abuse Rehabilitation Facility Substance Abuse Treatment, Children",Residential Treatment Facilities +331L00000X,Blood Bank ,Suppliers +332000000X,Military/U.S. Coast Guard Pharmacy ,Suppliers +332100000X,Department of Veterans Affairs (VA) Pharmacy ,Suppliers +332800000X,Indian Health Service/Tribal/Urban Indian Health (I/T/U) Pharmacy ,Suppliers +332900000X,Non-Pharmacy Dispensing Site ,Suppliers +332B00000X,Durable Medical Equipment & Medical Supplies ,Suppliers +332BC3200X,Durable Medical Equipment & Medical Supplies Customized Equipment,Suppliers +332BD1200X,Durable Medical Equipment & Medical Supplies Dialysis Equipment & Supplies,Suppliers +332BN1400X,Durable Medical Equipment & Medical Supplies Nursing Facility Supplies,Suppliers +332BP3500X,Durable Medical Equipment & Medical Supplies Parenteral & Enteral Nutrition,Suppliers +332BX2000X,Durable Medical Equipment & Medical Supplies Oxygen Equipment & Supplies,Suppliers +332G00000X,Eye Bank ,Suppliers +332H00000X,"Eyewear Supplier (Equipment, not the service) ",Suppliers +332S00000X,Hearing Aid Equipment ,Suppliers +332U00000X,Home Delivered Meals ,Suppliers +333300000X,Emergency Response System Companies ,Suppliers +333600000X,Pharmacy ,Suppliers +3336C0002X,Pharmacy Clinic Pharmacy,Suppliers +3336C0003X,Pharmacy Community/Retail Pharmacy,Suppliers +3336C0004X,Pharmacy Compounding Pharmacy,Suppliers +3336H0001X,Pharmacy Home Infusion Therapy Pharmacy,Suppliers +3336I0012X,Pharmacy Institutional Pharmacy,Suppliers +3336L0003X,Pharmacy Long Term Care Pharmacy,Suppliers +3336M0002X,Pharmacy Mail Order Pharmacy,Suppliers +3336M0003X,Pharmacy Managed Care Organization Pharmacy,Suppliers +3336N0007X,Pharmacy Nuclear Pharmacy,Suppliers +3336S0011X,Pharmacy Specialty Pharmacy,Suppliers +335E00000X,Prosthetic/Orthotic Supplier ,Suppliers +335G00000X,Medical Foods Supplier ,Suppliers +335U00000X,Organ Procurement Organization ,Suppliers +335V00000X,Portable X-ray and/or Other Portable Diagnostic Imaging Supplier ,Suppliers +341600000X,Ambulance ,Transportation Services +3416A0800X,Ambulance Air Transport,Transportation Services +3416L0300X,Ambulance Land Transport,Transportation Services +3416S0300X,Ambulance Water Transport,Transportation Services +341800000X,Military/U.S. Coast Guard Transport ,Transportation Services +3418M1110X,"Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance, Ground Transport",Transportation Services +3418M1120X,"Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance, Air Transport",Transportation Services +3418M1130X,"Military/U.S. Coast Guard Transport Military or U.S. Coast Guard Ambulance, Water Transport",Transportation Services +343800000X,Secured Medical Transport (VAN) ,Transportation Services +343900000X,Non-emergency Medical Transport (VAN) ,Transportation Services +344600000X,Taxi ,Transportation Services +344800000X,Air Carrier ,Transportation Services +347B00000X,Bus ,Transportation Services +347C00000X,Private Vehicle ,Transportation Services +347D00000X,Train ,Transportation Services +347E00000X,Transportation Broker ,Transportation Services +363A00000X,Physician Assistant ,Physician Assistants & Advanced Practice Nursing Providers +363AM0700X,Physician Assistant Medical,Physician Assistants & Advanced Practice Nursing Providers +363AS0400X,Physician Assistant Surgical Technologist,Physician Assistants & Advanced Practice Nursing Providers +363L00000X,Nurse Practitioner ,Physician Assistants & Advanced Practice Nursing Providers +363LA2100X,Nurse Practitioner Acute Care,Physician Assistants & Advanced Practice Nursing Providers +363LA2200X,Nurse Practitioner Adult Health,Physician Assistants & Advanced Practice Nursing Providers +363LC0200X,Nurse Practitioner Critical Care Medicine,Physician Assistants & Advanced Practice Nursing Providers +363LC1500X,Nurse Practitioner Community Health,Physician Assistants & Advanced Practice Nursing Providers +363LF0000X,Nurse Practitioner Family,Physician Assistants & Advanced Practice Nursing Providers +363LG0600X,Nurse Practitioner Gerontology,Physician Assistants & Advanced Practice Nursing Providers +363LN0000X,Nurse Practitioner Neonatal,Physician Assistants & Advanced Practice Nursing Providers +363LN0005X,"Nurse Practitioner Neonatal, Critical Care",Physician Assistants & Advanced Practice Nursing Providers +363LP0200X,Nurse Practitioner Pediatrics,Physician Assistants & Advanced Practice Nursing Providers +363LP0222X,"Nurse Practitioner Pediatrics, Critical Care",Physician Assistants & Advanced Practice Nursing Providers +363LP0808X,Nurse Practitioner Psych/Mental Health,Physician Assistants & Advanced Practice Nursing Providers +363LP1700X,Nurse Practitioner Perinatal,Physician Assistants & Advanced Practice Nursing Providers +363LP2300X,Nurse Practitioner Primary Care,Physician Assistants & Advanced Practice Nursing Providers +363LS0200X,Nurse Practitioner School,Physician Assistants & Advanced Practice Nursing Providers +363LW0102X,Nurse Practitioner Women's Health,Physician Assistants & Advanced Practice Nursing Providers +363LX0001X,Nurse Practitioner Obstetrics & Gynecology,Physician Assistants & Advanced Practice Nursing Providers +363LX0106X,Nurse Practitioner Occupational Health,Physician Assistants & Advanced Practice Nursing Providers +364S00000X,Clinical Nurse Specialist ,Physician Assistants & Advanced Practice Nursing Providers +364SA2100X,Clinical Nurse Specialist Acute Care,Physician Assistants & Advanced Practice Nursing Providers +364SA2200X,Clinical Nurse Specialist Adult Health,Physician Assistants & Advanced Practice Nursing Providers +364SC0200X,Clinical Nurse Specialist Critical Care Medicine,Physician Assistants & Advanced Practice Nursing Providers +364SC1501X,Clinical Nurse Specialist Community Health/Public Health,Physician Assistants & Advanced Practice Nursing Providers +364SC2300X,Clinical Nurse Specialist Chronic Care,Physician Assistants & Advanced Practice Nursing Providers +364SE0003X,Clinical Nurse Specialist Emergency,Physician Assistants & Advanced Practice Nursing Providers +364SE1400X,Clinical Nurse Specialist Ethics,Physician Assistants & Advanced Practice Nursing Providers +364SF0001X,Clinical Nurse Specialist Family Health,Physician Assistants & Advanced Practice Nursing Providers +364SG0600X,Clinical Nurse Specialist Gerontology,Physician Assistants & Advanced Practice Nursing Providers +364SH0200X,Clinical Nurse Specialist Home Health,Physician Assistants & Advanced Practice Nursing Providers +364SH1100X,Clinical Nurse Specialist Holistic,Physician Assistants & Advanced Practice Nursing Providers +364SI0800X,Clinical Nurse Specialist Informatics,Physician Assistants & Advanced Practice Nursing Providers +364SL0600X,Clinical Nurse Specialist Long-Term Care,Physician Assistants & Advanced Practice Nursing Providers +364SM0705X,Clinical Nurse Specialist Medical-Surgical,Physician Assistants & Advanced Practice Nursing Providers +364SN0000X,Clinical Nurse Specialist Neonatal,Physician Assistants & Advanced Practice Nursing Providers +364SN0800X,Clinical Nurse Specialist Neuroscience,Physician Assistants & Advanced Practice Nursing Providers +364SP0200X,Clinical Nurse Specialist Pediatrics,Physician Assistants & Advanced Practice Nursing Providers +364SP0807X,"Clinical Nurse Specialist Psych/Mental Health, Child & Adolescent",Physician Assistants & Advanced Practice Nursing Providers +364SP0808X,Clinical Nurse Specialist Psych/Mental Health,Physician Assistants & Advanced Practice Nursing Providers +364SP0809X,"Clinical Nurse Specialist Psych/Mental Health, Adult",Physician Assistants & Advanced Practice Nursing Providers +364SP0810X,"Clinical Nurse Specialist Psych/Mental Health, Child & Family",Physician Assistants & Advanced Practice Nursing Providers +364SP0811X,"Clinical Nurse Specialist Psych/Mental Health, Chronically Ill",Physician Assistants & Advanced Practice Nursing Providers +364SP0812X,"Clinical Nurse Specialist Psych/Mental Health, Community",Physician Assistants & Advanced Practice Nursing Providers +364SP0813X,"Clinical Nurse Specialist Psych/Mental Health, Geropsychiatric",Physician Assistants & Advanced Practice Nursing Providers +364SP1700X,Clinical Nurse Specialist Perinatal,Physician Assistants & Advanced Practice Nursing Providers +364SP2800X,Clinical Nurse Specialist Perioperative,Physician Assistants & Advanced Practice Nursing Providers +364SR0400X,Clinical Nurse Specialist Rehabilitation,Physician Assistants & Advanced Practice Nursing Providers +364SS0200X,Clinical Nurse Specialist School,Physician Assistants & Advanced Practice Nursing Providers +364ST0500X,Clinical Nurse Specialist Transplantation,Physician Assistants & Advanced Practice Nursing Providers +364SW0102X,Clinical Nurse Specialist Women's Health,Physician Assistants & Advanced Practice Nursing Providers +364SX0106X,Clinical Nurse Specialist Occupational Health,Physician Assistants & Advanced Practice Nursing Providers +364SX0200X,Clinical Nurse Specialist Oncology,Physician Assistants & Advanced Practice Nursing Providers +364SX0204X,"Clinical Nurse Specialist Oncology, Pediatrics",Physician Assistants & Advanced Practice Nursing Providers +367500000X,"Nurse Anesthetist, Certified Registered ",Physician Assistants & Advanced Practice Nursing Providers +367A00000X,Advanced Practice Midwife ,Physician Assistants & Advanced Practice Nursing Providers +367H00000X,Anesthesiologist Assistant ,Physician Assistants & Advanced Practice Nursing Providers +372500000X,Chore Provider ,Nursing Service Related Providers +372600000X,Adult Companion ,Nursing Service Related Providers +373H00000X,Day Training/Habilitation Specialist ,Nursing Service Related Providers +374700000X,Technician ,Nursing Service Related Providers +3747A0650X,Technician Attendant Care Provider,Nursing Service Related Providers +3747P1801X,Technician Personal Care Attendant,Nursing Service Related Providers +374J00000X,Doula ,Nursing Service Related Providers +374K00000X,Religious Nonmedical Practitioner ,Nursing Service Related Providers +374T00000X,Religious Nonmedical Nursing Personnel ,Nursing Service Related Providers +374U00000X,Home Health Aide ,Nursing Service Related Providers +376G00000X,Nursing Home Administrator ,Nursing Service Related Providers +376J00000X,Homemaker ,Nursing Service Related Providers +376K00000X,Nurse's Aide ,Nursing Service Related Providers +385H00000X,Respite Care ,Respite Care Facility +385HR2050X,Respite Care Respite Care Camp,Respite Care Facility +385HR2055X,"Respite Care Respite Care, Mental Illness, Child",Respite Care Facility +385HR2060X,"Respite Care Respite Care, Mental Retardation and/or Developmental Disabilities, Child",Respite Care Facility +385HR2065X,"Respite Care Respite Care, Physical Disabilities, Child",Respite Care Facility +390200000X,Student in an Organized Health Care Education/Training Program ,"Student, Health Care" +405300000X,Prevention Professional ,Other Service Providers +NI,No information, +UN,Unknown, +OT,Other, diff --git a/curated_data/resultunit_manualcuration.csv b/curated_data/resultunit_manualcuration.csv new file mode 100644 index 0000000..03a899c --- /dev/null +++ b/curated_data/resultunit_manualcuration.csv @@ -0,0 +1,162 @@ +FREQ,RESULT_UNIT,UCUM_CODE,UCUM_DESCRIPTIVE_TEXT +1123,#,OT,Other +2221154,/HPF,OT,Other +46026,/LPF,OT,Other +80338,/UL,OT,Other +18887,[IU]/ML,[IU]/mL,international unit per milliliter +10151,10*3/UL,10*3/uL,thousand per microliter +2140,10*6/UL,10*6/uL,million per microliter +2419,10^3,OT,Other +6451,10^3/UL,10*3/uL,thousand per microliter +1992,1000/UL,10*3/uL,thousand per microliter +10563,10E9/L,OT,Other +11642,AI,OT,Other +3141,AU,OT,Other +174064,BPM,OT,Other +2842,CC,OT,Other +689040,CELLS/UL,OT,Other +1366532,CM,OT,Other +5972,CM (<=0.5),OT,Other +68355,CM (<=4.4),OT,Other +73132,CM (<=5.3),OT,Other +5161,cm (2.0-2.8,OT,Other +77157,cm (2.0-3.5,OT,Other +77545,cm (2.4-4.2,OT,Other +5148,cm (2.7-3.3,OT,Other +76747,cm (5.6-8.6,OT,Other +5149,cm (7.1-7.9,OT,Other +74761,CM2,OT,Other +83688,CM2 (<=18),OT,Other +26975,CM3,OT,Other +1010,CMM,OT,Other +14348,COPIES/ML,OT,Other +7561,DEG,OT,Other +3020,E10 CELLS,OT,Other +3018,E6 CELLS,OT,Other +3018,E6 CELLS/KG,OT,Other +6314023,FL,fL,femtoliter +7565,G,g,gram +1577,G/24 H,g/(24.h),gram per 24 hour +8288964,G/DL,g/dL,gram per deciliter +7412,G/M2,OT,Other +3149751,GM/DL,g/dL,gram per deciliter +1153,GMS,OT,Other +1122,GPL,g/L,gram per liter +8459,GPL/ML,OT,Other +1678,H,h,hour +1678,H,H,Henry +11111,IN,OT,Other +7319,INDEX,OT,Other +2123,INDEX VALUE,OT,Other +1756,INR,OT,Other +20221,IU/L,OT,Other +120014,IU/ML,OT,Other +3099,K/CMM,OT,Other +4645,K/CUMM,OT,Other +1376,K/MM3,OT,Other +15274855,K/UL,OT,Other +4353,KG,kg,kilogram +39896,KU/L,OT,Other +11408,LBS,OT,Other +1210,LOG IU/ML,OT,Other +1325,M,OT,Other +1118508,M/S,OT,Other +3045285,M/UL,mU/L,milli enzyme unit per liter +169686,M2,m2,square meter +1183,MCG/24 H,OT,Other +280255,MCG/DL,OT,Other +4738,mcg/dL (cal,OT,Other +2570,MCG/KG/MIN,OT,Other +9586,MCG/L,OT,Other +16459,mcg/mg crea,OT,Other +162148,MCG/ML,OT,Other +101181,MCI,OT,Other +21359,MCMOL/L,OT,Other +311906,MCU/ML,OT,Other +39352,MEG/L,OT,Other +150846,MEQ/L,OT,Other +2984,METER,OT,Other +29104,METS,OT,Other +1192,MG/24 H,mg/(24.h),milligram per 24 hour +15652,MG/24 HRS,mg/(24.h),milligram per 24 hour +2007,MG/24HR,mg/(24.h),milligram per 24 hour +24352219,MG/DL,mg/dL,milligram per deciliter +267299,mg/dL (calc,mg/dL,milligram per deciliter +2975,MG/G CREAT,mg/g{creat},milligram per gram of creatinine +8998,MG/GRAM,mg/g,milligram per gram +24634,MG/L,mg/L,milligram per liter +3017,MIL/UL,m[IU]/mL,milli international unit per milliliter +167079,MILLION/UL,OT,Other +61834,MIN,min,minute +116367,MIU/L,m[IU]/L,milli international unit per liter +61043,MIU/ML,m[IU]/mL,milli international unit per milliliter +151959,ML,mL,milliliter +4050252,ML/MIN,mL/min,milliliter per minute +6362259,ML/MIN/1.73,OT,Other +460610,mL/min/1.73,OT,Other +1517,ml/min/1.73,OT,Other +17822,MLS,OT,Other +8022,MM,OT,Other +14015,MM/H,mm/h,millimeter per hour +98074,MM/HR,mm/h,millimeter per hour +1285125,MMHG,mm[Hg],millimeter of mercury +3923,MMOL/24 HRS,mmol/(24.h),millimole per 24 hour +14747546,MMOL/L,mmol/L,millimole per liter +20357,MOS/KG,mosm/kg,milliosmole per kilogram +23350,MOSMOL/KG,mosm/kg,milliosmole per kilogram +1536,MPH,OT,Other +1071,MPL,OT,Other +8108,MPL/ML,OT,Other +26039,MS,OT,Other +152378,MSEC,OT,Other +21140,MU/ML,mU/mL,milli enzyme unit per milliliter +148859,NG/DL,ng/dL,nanogram per deciliter +1181293,NG/ML,ng/mL,nanogram per millliiter +37283,NG/ML DDU,OT,Other +1658,NG/ML FEU,OT,Other +26640416,NI,NI,No information +1628,NMBCE/L,OT,Other +11552,NMOL/L,nmol/L,nanomole per liter +25756,NMOL/ML,nmol/mL,nanomole per milliliter +1123,NMOLES,OT,Other +19633313,PERCENT,%,percent +3233164,PG,pg,picogram +273580,PG/ML,pg/mL,picogram per milliliter +8363,RATIO,OT,Other +10252,RNA COPIES/,OT,Other +33807,S,S,Siemens +33807,S,s,second +505524,SEC,s,second +1184,SECONDS,s,second +5070,SECS,s,second +4564,SGU,OT,Other +4340,SMU,OT,Other +1647,T/CMM,OT,Other +1529,TH/CMM,OT,Other +5180,TH/UL,OT,Other +2212,THOU/UL,OT,Other +2239,THOUS,OT,Other +4073,THOUS/UL,OT,Other +335245,THOUSAND/UL,OT,Other +134897,TITER,OT,Other +7394762,U/L,U/L,enzyme unit per liter +90061,U/ML,U/mL,enzyme unit per milliliter +9425,UG/DL,ug/dL,microgram per deciliter +1636,UG/L,ug/L,microgram per liter +5618,UG/MG,OT,Other +1874,UG/ML,ug/mL,microgram per milliliter +34358,UIU/ML,u[IU]/mL,micro international unit per milliliter +39742,UL,U/L,enzyme unit per liter +28578,UMOL/L,umol/L,micromole per liter +5970,UNIT/L,U/L,enzyme unit per liter +2793,UNITS,OT,Other +4069,UNITS/L,U/L,enzyme unit per liter +2341,WEEKS,wk,week +1707,X10 3/UL,OT,Other +1205,X10.E3/UL,OT,Other +4231,X10^3,OT,Other +6873,X10^3/UL,OT,Other +2409,X1000,OT,Other +1333,X10-3/UL,OT,Other +15873,X10E3/UL,OT,Other +2272,X10E6/UL,OT,Other diff --git a/curated_data/route_map.csv b/curated_data/route_map.csv new file mode 100644 index 0000000..60a6b8a --- /dev/null +++ b/curated_data/route_map.csv @@ -0,0 +1,207 @@ +ROUTE_NAME,CODE +adductor canal block,OT +Apheresis,EXTRACORPOREAL +Apply externally,OT +Arterial,INTRA_ARTERIAL +Base of the eyelashes,OT +Bladder Instillation,OT +Block,OT +Both Ears,OTIC +Both Eyes,INTRAOCULAR +Buccal,BUCCAL +Catheter Sheath,OT +Caudal Block,CAUDAL +Central,OT +central line irrigation,OT +Cervical,OT +Chest Tube,OT +Combination,OT +CONT CAUDAL INFUSION,CAUDAL +CONT INTRAARTER INF,OT +CONT INTRAOSSEOUS IF,OT +CONT INTRATHECAL INF,INTRATHECAL +CONT NEBULIZATION,OT +CONT SUBCUTAN INFUSI,SUBCUTANEOUS +Contin. Intraosseous Infusion,OT +Contin. Intrathecal Infusion,INTRATHECAL +Contin. Subcutaneous Infusion,SUBCUTANEOUS +Contin.Intra-Arterial Infusion,INTRA_ARTERIAL +continuous adductor canal block,OT +Continuous Caudal Infusion,CAUDAL +CONTINUOUS EPIDURAL,EPIDURAL +CONTINUOUS INFILTRAT,OT +continuous infraclavicular,OT +CONTINUOUS INFUSION,OT +continuous interscalene,OT +Continuous Intraperitoneal Infusion,INTRAPERITONEAL +Continuous IV Infusion,INTRAVENOUS +Continuous Nebulization,OT +CRRT,OT +cytapheresis,OT +deal,OT +dental,DENTAL +Dialysis,OT +Does not apply,UN +Each Nostril,NASAL +Endotracheal,OT +Epidural,EPIDURAL +EXTRACORPOREAL,EXTRACORPOREAL +Feeding,OT +Feeding Tube,OT +Flush,OT +Gastrostomy Tube,GASTROSTOMY +GU irrigant,OT +Gums,GINGIVAL +HAND BULB NEBULIZER,OT +Hemodialysis,OT +hemodialysis port injection,OT +IM/PO DISCRETIONARY,OT +Implant,OT +In Vitro,OT +Infiltration,OT +infraclavicular,OT +Inhalation,OT +Injection,OT +Insufflation,OT +Intercatheter,OT +interscalene,INTRAMUSCULAR +Intra-amniotic,INTRAAMNIOTIC +Intra-arterial,INTRA_ARTERIAL +Intra-articular,INTRA_ARTICULAR +INTRABURSAL,INTRABURSAL +Intracameral,INTRACAMERAL +Intracardiac,INTRACARDIAC +Intra-catheter,OT +Intracavernosal,INTRACAVERNOUS +INTRACAVITY,BODY_CAVITY +Intradermal,INTRADERMAL +INTRADETRUSOR,OT +Intraductal,INTRADUCTAL +Intralesional,INTRALESIONAL +Intra-Lesional,INTRALESIONAL +INTRALUMBAR,INTRASPINAL +Intra-lymphatic,INTRALYMPHATIC +Intramuscular,INTRAMUSCULAR +Intranasal,INTRASINAL +Intraocular,INTRAOCULAR +intraocular injection,INTRAOCULAR +intraocular irrigation,INTRAOCULAR +Intraosseous,INTRAOSSEOUS +INTRAPERICARDIAL,INTRAPERICARDIAL +Intraperitoneal,INTRAPERITONEAL +Intrapleural,INTRAPLEURAL +Intraspinal,INTRASPINAL +Intrasynovial,INTRASYNOVIAL +Intrathecal,INTRATHECAL +Intrathoracic,INTRATHORACIC +Intratracheal,INTRATRACHEAL +intratympanic,INTRATYMPANIC +INTRA-URETHRAL,URETHRAL +Intrauterine,INTRAUTERINE +INTRAVARICEAL,INTRAVENOUS +Intravenous,INTRAVENOUS +intravenous push,OT +INTRAVENTRICULAR,INTRACEREBROVENTRICULAR +Intravesical,INTRAVESICAL +intravesical irrigation,INTRAVESICAL +INTRAVITREAL,INTRAVITREAL +Iontophoresis,OT +IPPB,OT +Irrigation,OT +JUXTASCLERAL,OT +Laryngotracheal,OT +Left Ear,OTIC +Left Eye,INTRAOCULAR +Local Infiltration,OT +local intranasal application,OT +Misc.(Non-Drug; Combo Route),OT +MISCELLANEOUS,OT +Mouth/Throat,ORAL +Mucous Membrane,OT +Nares,NASAL +Nasal,NASAL +Nasogastric,NASOGASTRIC +nasogastric tube,NASOGASTRIC +Nebulization,OT +NEBULIZATION -UNSPEC,OT +NOT APPLICABLE,UN +O2 AEROSOLIZATION,OT +Ommaya Reservior,OT +One Nostril,NASAL +Ophthalmic,OPHTHALMIC +Oral,ORAL +ORAL/IV,OT +ORAL/RECTAL/IM,OT +ORAL/RECTAL/IV,OT +Osteochondrial,OT +Otic,OTIC +PEG Tube,PERCUTANEOUS +PEG-J tube,PERCUTANEOUS +Per Corpak Tube,OT +Per Dobhoff Tube,NASOGASTRIC +Per G Tube,GASTROSTOMY +Per J Tube,JEJUNOSTOMY +Per NG tube,NASOGASTRIC +Per OG Tube,OROGASTRIC +Percutaneous,PERCUTANEOUS +Perfusion,OT +Perianal,OT +periarticular,OT +Peri-articular,PERIARTICULAR +PERIBULBAR,PERIBULBAR +Pericapsular,OT +Perineal,OT +perineural injection,PERINEURAL +Periodontal,PERIODONTAL +Periodontal pocket,OT +Peripheral,OT +Peripheral Arterial Line,OT +peripheral nerve block,OT +Peripheral Nerve Cath,OT +Peritoneal catheter,OT +Pump Refill,OT +Rectal,RECTAL +RECTAL/ORAL DISCRET,OT +RETROBULBAR,RETROBULBAR +Retrobulbular,RETROBULBAR +Right Ear,OTIC +Right Eye,INTRAOCULAR +scalp,OT +Scratch,OT +SEE ADMIN INSTRUCTIONS,UN +Service,OT +Sheath,OT +Sub lesionally,SUBLESIONAL +SUBCONJUCTIVAL,SUBCONJUNCTIVAL +Subconjunctival,SUBCONJUNCTIVAL +Subcutaneous,SUBCUTANEOUS +Subdermal,OT +Subgingival-Local,SUBGINGIVAL +Sublesional,SUBLESIONAL +Sublesionally,SUBLESIONAL +Sublingual,SUBLINGUAL +Submucosal anal canal,OT +SUBMUCOSAL INJ,SUBMUCOSAL +Submucosal Injection,SUBMUCOSAL +SubQ Pump,OT +subretinal,OT +SUB-TENON,OT +Swish & Spit,ORAL +Swish & Swallow,ORAL +TENDON SHEATH INJ.,OT +Tendon Sheath Injection,OT +Test,OT +Topical,TOPICAL +Tracheal Tube,OT +Transdermal,TRANSDERMAL +Translingual,OT +TRANSTRACHEAL,TRANSTRACHEAL +TRANSURETHRAL,TRANSURETHRAL +Tube,OT +Umbilical Artery Cath,OT +Unknown,UN +Urethral,URETHRAL +urinary catheter irrigation,OT +Vaginal,VAGINAL +venous line irrigation,OT +Wound irrigation,OT diff --git a/curated_data/specimen_source_map.csv b/curated_data/specimen_source_map.csv new file mode 100644 index 0000000..392e3f2 --- /dev/null +++ b/curated_data/specimen_source_map.csv @@ -0,0 +1,492 @@ +SPECIMEN_SOURCE_NAME,CODE +,^BPU +,^BPU.AUTOLOGOUS +,^BPU.PLATELET_PHERESIS +,^CANCER +,^EGG_DONOR +,^EMBRYO +,^FATHER +Fetus,^FETUS +,^GUARDIAN_OR_LEGALLY_AUTHORIZED_REPRESENTATIVE +,^MOTHER +,^MUSHROOM_SPECIMEN +,^PATIENT +,^PLANT_SPECIMEN +,^POPULATION +,^STEM_CELL_PRODUCT +,^TICK +,{SETTING} +,ABDOMEN.FNA +Abscess,ABSCESS +"Adrenal Gland,Left",ADRENAL_GLAND +"Adrenal Gland,Right",ADRENAL_GLAND +,AIR +Amniotic Fluid,AMNIO_FLD +,AMNIO_FLD_CELLS +,AMNIO_FLD_CVS +Anus,ANAL +,ANOGENITAL +,ANORECTAL +,ANORECTAL_ISOLATE +,ANORECTAL_STOOL +Aorta,AORTA.ROOT +,ASP +,B_CELLS+MONOCYTES +,BAL +Bartholin Cyst,BARTHOLIN_CYST +,BBL +Biliary Fluid,BIL_FLD +Blood,BLD +Blood Clot,BLD +"Blood, Arterial",BLDA +"Blood,Line Draw",BLD +"Blood,Peripheral",BLDP +,BLD_MC +,BLD.BUFFY_COAT +,BLD.DOT +,BLD.POS_GROWTH +,BLD_BONE_MAR +,BLD_BONE_MAR^DONOR +,BLD_TISS +,BLD_TISS_SAL +,BLD_TISS^DONOR +,BLD_URINE +,BLD^BPU +,BLD^CONTROL +,BLD^DONOR +,BLD^FATHER +,BLD^FETUS +,BLD^NEWBORN +,BLD^PATIENT +,BLD+INHL_GAS +,BLDA +,BLDA+INHL_GAS +,BLDC +,BLDC^FETUS +,BLDC+INHL_GAS +,BLDCO +,BLDCOA +,BLDCOMV +,BLDCOV +,BLDCRRT +,BLDMV +,BLDMV+INHL_GAS +,BLDP +,BLDV +,BLDV+INHL_GAS +,BODY_FLD +,BODY_FLD.SPUN +,BODY_FLD^FETUS +,BODY_FLD+SER_PLAS +Bone,BONE +Bone Specimen,BONE +,BONE_MAR +Bone Marrow,BONE_MARROW +,BONE^DONOR +Brain,BRAIN +Breast,BREAST +"Breast,Left",BREAST +"Breast,Right",BREAST +"Breast Cyst,Left",BREAST +"Breast Cyst,Right",BREAST +"Nipple Smear,Left",BREAST +"Nipple Smear,Right",BREAST +,BREAST_CANCER_SPECIMEN +,BREAST_TUMOR +,BREAST.DUCTAL_LAVAGE +,BREAST.FNA +Bronchial Alveolar Lavage,BRONCHIAL +"Bronchial Alveolar Lavage,LLL",BRONCHIAL +"Bronchial Alveolar Lavage,LUL",BRONCHIAL +"Bronchial Alveolar Lavage,RLL",BRONCHIAL +"Bronchial Alveolar Lavage,RML",BRONCHIAL +"Bronchial Alveolar Lavage,RUL",BRONCHIAL +Bronchial Wash(Specify Site),BRONCHIAL +Bronchial Washing LLL,BRONCHIAL +Bronchial Washing LUL,BRONCHIAL +Bronchial Washing RLL,BRONCHIAL +Bronchial Washing RML,BRONCHIAL +Bronchial Washing RUL,BRONCHIAL +Bronchus,BRONCHIAL +Bronchial Brush (Specify Site),BRONCHIAL_BRUSH +"Bronchial Brush,LLL",BRONCHIAL_BRUSH +"Bronchial Brush,LUL",BRONCHIAL_BRUSH +"Bronchial Brush,RLL",BRONCHIAL_BRUSH +"Bronchial Brush,RML",BRONCHIAL_BRUSH +"Bronchial Brush,RUL",BRONCHIAL_BRUSH +Buccal Mucosa,BUCCAL +,BUCCAL_SMEAR +,BURN +,BURSA_OF_FABRICIUS +,CALCULUS +,CANCER_SPECIMEN +,CELLS.XXX +Cervix,CERVIX +,CHEESE +,CNJT +,CNL +,COL +Colon,COLON +,COLORECTAL_CANCER_SPECIMEN +,CONTACT_LENS +,CORONARY_SINUS +,CRN +,CSF +,CSF.SPUN +,CTP +,CURRENT_SAMPLE +,CVM +,CVS +,CVX +,CVX_VAG +,DAIRY_PRODUCT +,DEEP_TISSUE.FNA +,DENTIN +Dialysate,DIAL_FLD +,DIAL_FLD_PRT +,DIAL_FLD_PRT+SER_PLAS +,DIAL_FLD.SPUN +,DIAL_FLD+SER_PLAS +,DOSE +,DRAIN +,DUCTUS_ARTERIOSUS +,DUOD_FLD +,DUOD_FLD_GAST_FLD +Ear,EAR +Ear Middle,EAR +Middle Ear,EAR +Auditory Canal,EAR +Auditory Canal Internal,EAR +Tympanic Cavity,EAR +Eustachian Tube,EAR +,EAR_FLUID +,EGG +,EGGYLK +,ENDOCERVICAL_BRUSH +Endometrial,ENDOMET +Endocardium,ENDOMYOCARDIUM +,ENVIR +,ENVIRONMENTAL_SPECIMEN +Esophageal Brush,ESOPHAGEAL_BRUSH +,EXHL_GAS +,EXTRACELLULAR_FLD +,EXUDATE +Eye,EYE +Cornea,EYE +Corneal Scrapings,EYE +Conjunctiva,EYE +Eye Drainage,EYE +Eyelid,EYE +Contact Lens Solution,OT +,FACILITY +,FEATHER +,FEED +,FIBROBLASTS +,FIBROBLASTS^CONTROL +,FLU.NONBIOLOGICAL +,FOOD +,GAS +,GAST_FLD +,GENITAL +,GENITAL_FLD +,GENITAL_LOC +,GENITAL_MUC +,GI_CNT +,GLUCOSE_METER_DEVICE +Hair,HAIR +,HEART.ATRIUM.LEFT +,HEART.ATRIUM.RIGHT +,HEART.ATRIUM.RIGHT.HIGH +,HEART.ATRIUM.RIGHT.LOW +,HEART.ATRIUM.RIGHT.MID +,HEART.VENTRICLE.LEFT +,HEART.VENTRICLE.RIGHT +,HEART.VENTRICLE.RIGHT.OUTFLOW_TRACT +,HEMATOPOIETIC_PROGENITOR_CELLS^BPU +,HYPERAL_SOLUTION +,IMPLANT_DEVICE +,INDEX_CASE^COMPARISON_CASE +,INGESTA +,INHL_GAS +,INITIAL_SAMPLE +,INTRATHECAL_SPACE +,INTRAVASCULAR_SPACE.XXX +,ISOLATE +,ISOLATE.MENINGITIS +,ISOLATE.PNEUMONIA +,ISOLATE.UTI +,ISOLATE+SER +Kidney,KIDNEY +Kidney Basin Fluid,KIDNEY +"Kidney, Left",KIDNEY +"Kidney, Right",KIDNEY +,KIDNEY.FNA +"Line,Arterial",LINE +,LITTER +Liver,LIVER +,LIVER.FNA +,LIVER^FETUS +Lung,LUNG +Lung LLL,LUNG +Lung LUL,LUNG +Lung RLL,LUNG +Lung RML,LUNG +Lung RUL,LUNG +Pulmonic,LUNG +,LUNG_TISS +,LUNG.FNA +Lymph Node (Specify),LYMPH_NODE +,LYMPH_NODE.FNA +,LYMPHOBLASTS +,MECONIUM +,MESORECTUM +Breast Milk,MILK +,MLK.RAW +,MMLK +Mouth,MOUTH +Oral Cavity,MOUTH +Oropharynx,MOUTH +Gingiva,MOUTH +Uvula,MOUTH +Mouth Washing,MOUTH +Mandible,MOUTH +Tongue,MOUTH +Teeth/Tooth,MOUTH +Muscle,MUSCLE +,NAIL +,NASAL_FLUID +,NBS_CARD +,NECK_MASS.FNA +,NIPPLE_DISCHARGE +Nose,NOSE +Nasal,NOSE +Nasal Wash,NOSE +Nasopharyngeal Aspirate,NOSE +Nasopharyngeal Swab,NOSE +Turbinate,NOSE +,NOSE_TRAC +,NPH +,OCULAR_FLD +Ovary,OVARY +Pancreas,PANCREAS +"Pancreas,Body",PANCREAS +"Pancreas,Head",PANCREAS +"Pancreas,Neck",PANCREAS +"Pancreas,Tail",PANCREAS +Pancreatic Fluid,PANCREAS +Pancreatic Pseudocyst,PANCREAS +,PANCREAS.FNA +,PAROTID.FNA +,PELVIS.FNA +Penis,PENIS +Penile Discharge,PENIS +Penile Lesion,PENIS +Pericardial Fluid,PERICARD_FLD +Pericardial,PERICARD_FLD +Pericardium,PERICARD_FLD +,PERICARD_FLD.SPUN +,PERICARD_FLD+SER_PLAS +Peritoneal Fluid,PERITON_FLD +Ascitic Fluid,PERITON_FLD +,PERITON_FLD.SPUN +,PERITON_FLD+SER_PLAS +Peritoneal Cavity,PERITONEUM +Peritoneal Dialysate Fluid,PERITONEUM +Peritoneal Lavage,PERITONEUM +Peritoneum,PERITONEUM +Pharynx,PHARYNX +Placenta,PLACENT +,PLANT +,PLAS +,PLAS.CFDNA +,PLAS_BLD +,PLAS_RBC +,PLAS^DONOR +,PLAS+CSF +,PLAS+URINE +,PLASA +,PLASV +Platelets,PLATELETS +Pleural Cavity,PLEURA +"Pleural,Left",PLEURA +"Pleural,Right",PLEURA +Pleural Fluid,PLR_FLD +,PLR_FLD.SPUN +,PLR_FLD^FETUS +,POC +,PPP +,PPP_BLD +,PPP^CONTROL +,PPP^FETUS +,PPP^POOL +,PREPUTIAL_WASH +Prostate,PROSTATE +,PROSTATE_CANCER.XXX +,PROSTATE_TUMOR +,PROSTATE.FNA +,PROSTATIC_FLD +,PROVIDER +,PRP +,PRP^CONTROL +,PRP^DONOR +,PULMONARY_ARTERY.LEFT +,PULMONARY_ARTERY.MAIN +,PULMONARY_ARTERY.RIGHT +,PULMONARY_WEDGE +Pus,PUS +Rbc,RBC +,RBC^BPU +,RBC^CONTROL +,RBC^DONOR +,RBC^FETUS +,RBC^PATIENT +,RBCCO +Rectum,RECTUM +Rectal Abscess,RECTUM +Rectal Swab,RECTUM +,REFERENCE_LAB_TEST +,REPORT +,REPOSITORY +,RESPIRATORY +,RESPIRATORY.LOWER +,RESPIRATORY.UPPER +,RETIC +,RH_IMMUNE_GLOBULIN +Saliva,SALIVA +,SALIVARY_GLAND.FNA +Semen,SEMEN +,SEMEN+CVM +,SEMIN_PLAS +,SER +,SER_BLD +,SER_PLAS +,SER_PLAS.ULTRACENTRIFUGATE +,SER_PLAS_BLD +,SER_PLAS_BLDV +,SER_PLAS_URINE +,SER_PLAS^BPU +,SER_PLAS^DONOR +,SER_PLAS^FETUS +,SER_PLAS^NORMAL_CONTROL +,SER_PLAS+CSF +,SER_PLAS+PLR_FLD +,SER_PLAS+STOOL +,SER_PLAS+SYNV_FLD +,SER^CONTROL +,SER^DONOR +,SER+BLD +,SER+CSF +,SER+PLAS +,SER+RBC^CONTROL +,SER+SALIVA +Sinus,SINUS +Skin,SKIN +Skin Biopsy,SKIN +Skin Scrapings,SKIN +"Skin Swab,Burn Unit",SKIN +"Skin Swab,Marrow Transplant",SKIN +,SKIN_MELANOMA +,SMALL_INTES_FIXED +,SMPLS +,SOFT_TISSUE.FNA +,SPECIMEN +,SPERMATOZOA +,SPIRAL_COLON +Spleen,SPLEEN +,SPTC +,SPTT +Sputum,SPUTUM +"Sputum,Cystic Fibrosis",SPUTUM +,SPUTUM_BRONCHIAL +,SPUTUM_GAST_FLD +Stomach,STOMACH +Gastric,STOMACH +Gastric Aspirate,STOMACH +Pylorus,STOMACH +,STOMACH_CANCER +Feces,STOOL +,STOOL.DRIED +,STOOL.WET +,STUDY +,SUBMANDIBULAR.FNA +,SUPERFICIAL_TISSUE.FNA +Sweat,SWEAT +Synovial Fluid,SYNV_FLD +Synovial,SYNV_FLD +Synovium,SYNV_FLD +,SYNV_FLD.SPUN +,TEAR +Testis,TESTIS +Esophageal,THRT +Esophagus,THRT +Epiglottis,THRT +Thyroid,THYROID +"Thyroid,Left",THYROID +"Thyroid,Right",THYROID +,THYROID.FNA +,TISS +,TISS_FAT +,TISS_FIXED +,TISS.FNA +,TISS^CONTROL +,TISS^FETUS +Tissue,TISSUE +Tissue Specimen,TISSUE +,TISSUE_CHIP +,TISSUE_CORE +,TLGI_TSMI +,TPN +Trachea,TRAC +Tracheal Aspirate,TRAC +Tracheostomy Site,TRAC +Transtracheal Aspirate,TRAC +,TRACHEAL_SWAB +,TROPHOBLASTS +,TSMI +,TUMOR +Ulcer,ULC +,UNK_SUB +Urethra,URETHRA +Urinary Bladder,URINARY_BLADDER +Urine,URINE +Urine Catheter,URINE +Urine Straight Catheter,URINE +"Urine, Foley Catheter",URINE +"Urine, Outpatient",URINE +"Urine,Clean Catch",URINE +"Urine,Nephrostomy",URINE +"Urine,Renal Transplant",URINE +"Urine,Suprapubic",URINE +,URINE_SED +,URINE^FETUS +,URINE+SER_PLAS +Uterus,UTERUS +Myometrium,UTERUS +Vaginal,VAG +Vaginal-Cpt,VAG +Vaginal/Cervical,VAG +Rectal Vaginal,VAG+RECTUM +,VENA_CAVA.INFERIOR +,VENA_CAVA.SUPERIOR +Vitreous Fluid,VITR_FLD +,VITR_FLD.SPUN +,VOMITUS +,VP_SHUNT +,WATER +,WBC +,WBC.DNA+PLAS.CFDNA +,WBC^CONTROL +,WHEY +Wound,WOUND +,WOUND.DEEP +,WOUND.SHLW +,XXX +,XXX_MC +,XXX.BODY_FLUID +,XXX.SWAB +,XXX.TISSUE +,XXX^DONOR +,NI +,UN +Other (Specify),OT +Other Specimen,OT diff --git a/curated_data/unit_map.csv b/curated_data/unit_map.csv new file mode 100644 index 0000000..f69edb3 --- /dev/null +++ b/curated_data/unit_map.csv @@ -0,0 +1,818 @@ +CODE,UNIT_NAME +%,% +%{abnormal}, +%{activity}, +%{aggregation}, +%{at_60_min}, +%{bacteria}, +%{basal_activity}, +%{baseline}, +%{binding}, +%{blockade}, +%{blocked}, +%{bound}, +%{breakdown}, +%{cells}, +%{deficient}, +%{dose}, +%{excretion}, +%{Hb}, +%{hemolysis}, +%{index}, +%{inhibition}, +%{loss_AChR}, +%{loss}, +%{lysis}, +%{normal}, +%{penetration}, +%{pooled_plasma}, +%{positive}, +%{RBCs}, +%{reactive}, +%{recovery}, +%{reference}, +%{relative}, +%{residual}, +%{saturation}, +%{total}, +%{uptake}, +%{viable}, +%{vol}, +%{WBCs}, +/(12.h), +/[arb'U], +/[HPF], +/[IU], +/[LPF], +/{entity}, +/{OIF}, +/10*10, +/10*12, +/10*12{RBCs}, +/10*3, +/10*3{RBCs}, +/10*4{RBCs}, +/10*6, +/10*9, +/100, +/100{cells}, +/100{neutrophils}, +/100{spermatozoa}, +/100{WBCs}, +/a, +/cm[H2O], +/d, +/dL, +/g, +/g{creat}, +/g{Hb}, +/g{tot_nit}, +/g{tot_prot}, +/g{wet_tis}, +/h, +/kg, +/kg{body_wt}, +/L, +/m2, +/m3, +/mg, +/min, +/mL, +/mm, +/mmol{creat}, +/mo, +/s, +/U, +/uL, +/wk, +[APL'U], +[APL'U]/mL, +[arb'U], +[arb'U]/mL, +[AU], +[bdsk'U], +[beth'U], +[CFU], +[CFU]/L, +[CFU]/mL, +[Ch], +[cin_i], +[degF], +[dr_av], +[drp], +[foz_us], +[ft_i], +[gal_us], +[GPL'U], +[GPL'U]/mL, +[HPF], +[in_i], +[in_i'H2O], +[IU],Int'l Units +[IU]/(2.h), +[IU]/(24.h),Int'l Units/day +[IU]/10*9{RBCs}, +[IU]/d, +[IU]/dL,Int'l Units/100 mL +[IU]/g, +[IU]/g{Hb}, +[IU]/h,Int'l Units/hr +[IU]/kg, +[IU]/kg/d, +[IU]/L,Int'l Units/L +[IU]/L{37Cel}, +[IU]/mg{creat}, +[IU]/min,Int'l Units/min +[IU]/mL,Int'l Units/mL +[ka'U], +[knk'U], +[lb_av], +[LPF], +[mclg'U], +[mi_i], +[MPL'U], +[MPL'U]/mL, +[oz_av], +[oz_tr], +[pH], +[ppb], +[ppm], +[ppm]{v/v}, +[ppth], +[pptr], +[psi], +[pt_us], +[qt_us], +[sft_i], +[sin_i], +[syd_i], +[tbs_us], +[tb'U], +[todd'U], +[tsp_us], +[yd_i], +{#}, +{#}/[HPF], +{#}/[LPF], +{#}/{platelet}, +{#}/L, +{#}/min, +{#}/mL, +{#}/uL, +{absorbance}, +{activity}, +{AHF'U}, +{APS'U}, +{ARU}, +{beats}/min, +{binding_index}, +{CAE'U}, +{CAG_repeats}, +{cells}, +{cells}/[HPF], +{cells}/uL, +{CH100'U}, +{clock_time}, +{copies}, +{copies}/mL, +{copies}/ug, +{count}, +{CPM}, +{CPM}/10*3{cell}, +{delta_OD}, +{dilution}, +{Ehrlich'U}, +{Ehrlich'U}/(2.h), +{Ehrlich'U}/100.g, +{Ehrlich'U}/d, +{Ehrlich'U}/dL, +{EIA_index}, +{EIA_titer}, +{EIA'U}, +{EIA'U}/U, +{ELISA'U}, +{EV}, +{FIU}, +{fraction}, +{GAA_repeats}, +{genomes}/mL, +{Globules}/[HPF], +{GPS'U}, +{HA_titer}, +{IFA_index}, +{IFA_titer}, +{ImmuneComplex'U}, +{index_val}, +{INR}, +{ISR}, +{JDF'U}, +{JDF'U}/L, +{KCT'U}, +{KRONU'U}/mL, +{Log_copies}/mL, +{Log_IU}, +{Log_IU}/mL, +{Log}, +{Lyme_index_value}, +{M.o.M}, +{minidrop}/min, +{minidrop}/s, +{mm/dd/yyyy}, +{MPS'U}, +{MPS'U}/mL, +{mutation}, +{OD_unit}, +{Pan_Bio'U}, +{percentile}, +{phenotype}, +{ratio}, +{RBC}/uL, +{rel_saturation}, +{Rubella_virus}, +{s_co_ratio}, +{saturation}, +{shift}, +{spermatozoa}/mL, +{STDV}, +{titer}, +{TSI_index}, +{WBCs}, +10*12/L, +10*3, +10*3/L, +10*3/mL, +10*3/uL, +10*3{copies}/mL, +10*3{RBCs}, +10*4/uL, +10*5, +10*6, +10*6.[CFU]/L, +10*6.[IU], +10*6/(24.h), +10*6/kg, +10*6/L, +10*6/mL, +10*6/uL, +10*8, +10*9/L, +10*9/mL, +10*9/uL, +10.L/(min.m2), +10.L/min, +10.uN.s/(cm5.m2), +24.h, +A, +a, +A/m, +ag/{cell}, +atm, +bar,Bar +Bq, +cal, +Cel, +cg, +cL, +cm,cm +cm[H2O], +cm[H2O]/L/s, +cm[H2O]/s/m, +cm[Hg], +cm2,cm2 cust +cm2/s, +cP, +cSt, +d, +daL/min, +daL/min/m2, +dB, +deg, +deg/s, +dg, +dL, +dm, +dm2/s2, +dyn.s/(cm.m2), +dyn.s/cm, +eq, +eq/L, +eq/mL, +eq/mmol, +eq/umol, +erg, +eV, +F, +fg, +fL, +fm, +fmol, +fmol/g, +fmol/L, +fmol/mg, +fmol/mg{cyt_prot}, +fmol/mg{prot}, +fmol/mL, +g,g +g.m, +g.m/{beat}, +g/(100.g), +g/(12.h), +g/(24.h),g/day +g/(3.d), +g/(4.h), +g/(48.h), +g/(5.h), +g/(6.h), +g/(72.h), +g/(8.h){shift}, +g/{specimen}, +g/{total_output}, +g/{total_weight}, +g/cm3, +g/d, +g/dL,g/100 mL +g/g, +g/g{creat}, +g/g{globulin}, +g/g{tissue}, +g/h,g/hr +g/h/m2, +g/kg ,g/kg +g/kg/(8.h), +g/kg/(8.h){shift}, +g/kg/d,g/kg/day +g/kg/h, +g/kg/min, +g/L,g/L +g/m2,g/m2 +g/mg, +g/min, +g/mL,g/mL +g/mmol, +g/mol{creat}, +g{creat}, +g{Hb}, +g{total_nit}, +g{total_prot}, +g{wet_tissue}, +Gy, +H, +h, +Hz, +J, +J/L, +K, +K/W, +k[IU]/L, +k[IU]/mL, +kat, +kat/kg, +kat/L, +kcal,kcal +kcal/[oz_av], +kcal/d,kcal/day +kcal/h, +kcal/kg/(24.h), +kg, +kg.m/s, +kg/(s.m2), +kg/h, +kg/L, +kg/m2, +kg/m3, +kg/min, +kg/mol, +kg/s, +kL, +km, +kPa, +ks, +kU, +kU/g, +kU/L, +kU/L{class}, +kU/mL, +L,L +L,l +L/(24.h), +L/(8.h), +L/(min.m2), +L/d,L/day +L/h, +L/kg, +L/L,L/L +L/min,L/min +L/s, +L/s/s2, +lm, +lm.m2, +m,m +m/s, +m/s2, +m[IU]/L, +m[IU]/mL, +m2, +m2/s, +m3/s, +mA, +mbar, +mbar.s/L, +mbar/L/s, +meq,mEq +meq/(2.h), +meq/(24.h), +meq/(8.h), +meq/{specimen}, +meq/{total_volume}, +meq/d,mEq/day +meq/dL,mEq/100 mL +meq/g, +meq/g{creat}, +meq/h,mEq/hr +meq/kg,mEq/kg +meq/kg/h, +meq/L, +meq/m2, +meq/min, +meq/mL, +mg,mg +mg/(10.h), +mg/(12.h), +mg/(2.h), +mg/(24.h), +mg/(6.h), +mg/(72.h), +mg/(8.h), +mg/{collection}, +mg/{specimen}, +mg/{total_output}, +mg/{total_volume}, +mg/d,mg/day +mg/d/{1.73_m2}, +mg/dL, +mg/dL{RBCs}, +mg/g, +mg/g{creat}, +mg/g{dry_tissue}, +mg/g{feces}, +mg/g{tissue}, +mg/g{wet_tissue}, +mg/h,mg/hr +mg/kg,mg/kg +mg/kg/(8.h), +mg/kg/d,mg/kg/day +mg/kg/h,mg/kg/hr +mg/kg/min,mg/kg/min +mg/L,mg/L +mg/L{RBCs}, +mg/m2,mg/m2 +mg/m3, +mg/mg, +mg/mg{creat}, +mg/min,mg/min +mg/mL,mg/mL +mg/mmol, +mg/mmol{creat}, +mg/wk, +mg{FEU}/L, +min, +mL,mL +mL,ml +mL/(10.h), +mL/(12.h), +mL/(2.h), +mL/(24.h), +mL/(4.h), +mL/(5.h), +mL/(6.h), +mL/(72.h), +mL/(8.h), +mL/(8.h)/kg, +mL/[sin_i], +mL/{beat}, +mL/{beat}/m2, +mL/cm[H2O], +mL/d,mL/day +mL/dL, +mL/h,mL/hr +mL/kg,mL/kg +mL/kg/(8.h), +mL/kg/d,mL/kg/day +mL/kg/h,mL/kg/hr +mL/kg/min, +mL/m2, +mL/mbar, +mL/min, +mL/min/{1.73_m2}, +mL/min/m2, +mL/mm, +mL/s, +mL{fetal_RBCs}, +mm,mm +mm/h, +mm/min, +mm[H2O], +mm[Hg], +mm2, +mmol,mmol +mmol/(12.h), +mmol/(2.h), +mmol/(24.h), +mmol/(5.h), +mmol/(6.h), +mmol/(8.h), +mmol/{ejaculate}, +mmol/{specimen}, +mmol/{total_vol}, +mmol/d, +mmol/dL, +mmol/g, +mmol/g{creat}, +mmol/h, +mmol/h/mg{Hb}, +mmol/h/mg{prot}, +mmol/kg,mmol/kg +mmol/kg/(8.h), +mmol/kg/d,mmol/kg/day +mmol/kg/h, +mmol/kg/min, +mmol/L, +mmol/L{RBCs}, +mmol/m2, +mmol/min, +mmol/mmol, +mmol/mmol{creat}, +mmol/mmol{urea}, +mmol/mol, +mmol/mol{creat}, +mmol/s/L, +mo, +mol, +mol/kg, +mol/kg/s, +mol/L, +mol/m3, +mol/mL, +mol/mol, +mol/s, +mosm, +mosm/kg, +mosm/L, +mPa, +mPa.s, +Ms, +ms, +mU/g, +mU/g{Hb}, +mU/g{prot}, +mU/L, +mU/mg, +mU/mg{creat}, +mU/mL, +mU/mL/min, +mU/mmol{creat}, +mU/mmol{RBCs}, +mV, +N, +N.cm, +N.s, +ng,ng +ng/(24.h), +ng/(8.h), +ng/10*6, +ng/10*6{RBCs}, +ng/d, +ng/dL, +ng/g, +ng/g{creat}, +ng/h, +ng/kg,ng/kg +ng/kg/(8.h), +ng/kg/h, +ng/kg/min,ng/kg/min +ng/L, +ng/m2, +ng/mg, +ng/mg/h, +ng/mg{creat}, +ng/mg{prot}, +ng/min, +ng/mL, +ng/mL/h, +ng/mL{RBCs}, +ng/s, +ng/U, +ng{FEU}/mL, +NI, +nkat, +nL, +nm, +nm/s/L, +nmol, +nmol/(24.h), +nmol/d, +nmol/dL, +nmol/dL{GF}, +nmol/g, +nmol/g{creat}, +nmol/g{dry_wt}, +nmol/h/L, +nmol/h/mg{prot}, +nmol/L, +nmol/L/mmol{creat}, +nmol/L{RBCs}, +nmol/m/mg{prot}, +nmol/mg, +nmol/mg/h, +nmol/mg{creat}, +nmol/mg{prot}, +nmol/mg{prot}, +nmol/mg{prot}/h, +nmol/min, +nmol/min/10*6{cells}, +nmol/min/mg{Hb}, +nmol/min/mg{prot}, +nmol/min/mL, +nmol/mL, +nmol/mL/h, +nmol/mL/min, +nmol/mmol, +nmol/mmol{creat}, +nmol/mmol{creat}, +nmol/mol, +nmol/nmol, +nmol/s, +nmol/s/L, +nmol/umol{creat}, +nmol{ATP}, +nmol{BCE}, +nmol{BCE}/L, +ns, +nU/{RBC}, +nU/mL, +Ohm, +Ohm.m, +osm, +osm/kg, +osm/L, +OT, +Pa, +pA, +pg, +pg/{cell}, +pg/{RBC}, +pg/dL, +pg/L, +pg/mg, +pg/mg{creat}, +pg/mL, +pg/mm, +pkat, +pL, +pm, +pmol, +pmol/(24.h), +pmol/{RBC}, +pmol/d, +pmol/dL, +pmol/g, +pmol/h/mg{prot}, +pmol/h/mL, +pmol/L, +pmol/mg{prot}, +pmol/min, +pmol/min/mg{prot}, +pmol/mL, +pmol/mmol{creat}, +pmol/umol, +pmol/umol{creat}, +ps, +pT, +s, +S, +s/{control}, +Sv, +t, +T, +Torr, +U,Units +U/(10.g){feces}, +U/(12.h), +U/(2.h), +U/(24.h), +U/10, +U/10*10, +U/10*10{cells}, +U/10*12, +U/10*12{RBCs}, +U/10*6, +U/10*9, +U/d,Units/day +U/dL,Units/100 mL +U/g, +U/g{creat}, +U/g{Hb}, +U/g{protein}, +U/h,Units/hr +U/kg{Hb}, +U/L,Units/L +U/min,Units/min +U/mL,Units/mL +U/mL{RBCs}, +U/mmol{creat}, +U/s, +u[IU], +u[IU]/L, +u[IU]/mL, +U{25Cel}/L, +U{37Cel}/L, +ueq, +ueq/L, +ueq/mL, +ug,mcg +ug/(100.g), +ug/(24.h),mcg/day +ug/(8.h), +ug/[sft_i], +ug/{specimen}, +ug/d, +ug/dL, +ug/dL{RBCs}, +ug/g, +ug/g{creat}, +ug/g{dry_tissue}, +ug/g{dry_wt}, +ug/g{feces}, +ug/g{hair}, +ug/g{Hb}, +ug/g{tissue}, +ug/h,mcg/hr +ug/kg,mcg/kg +ug/kg/(8.h), +ug/kg/d,mcg/kg/day +ug/kg/h,mcg/kg/hr +ug/kg/min,mcg/kg/min +ug/L,mcg/L +ug/L/(24.h), +ug/L{RBCs}, +ug/m2,mcg/m2 +ug/m3, +ug/mg, +ug/mg{creat}, +ug/min, +ug/mL,mcg/mL +ug/mL{class}, +ug/mL{eqv}, +ug/mmol, +ug/mmol{creat}, +ug/ng, +ug{FEU}/mL, +ukat, +uL,mcL +uL/(2.h), +uL/h, +um, +um/s, +umol, +umol/(2.h), +umol/(24.h), +umol/(8.h), +umol/d, +umol/dL, +umol/dL{GF}, +umol/g, +umol/g{creat}, +umol/g{Hb}, +umol/h, +umol/kg, +umol/kg{feces}, +umol/L, +umol/L/h, +umol/L{RBCs}, +umol/mg, +umol/mg{creat}, +umol/min, +umol/min/g, +umol/min/g{mucosa}, +umol/min/g{prot}, +umol/min/L, +umol/mL, +umol/mL/min, +umol/mmol, +umol/mmol{creat}, +umol/mol, +umol/mol{creat}, +umol/mol{Hb}, +umol/umol, +umol/umol{creat}, +umol{BCE}/mol, +UN, +uOhm, +us, +uU/g, +uU/L, +uU/mL, +uV, +V, +Wb, +wk, diff --git a/etl_tasks.py b/etl_tasks.py new file mode 100644 index 0000000..b040196 --- /dev/null +++ b/etl_tasks.py @@ -0,0 +1,965 @@ +'''etl_tasks -- Source-agnostic Luigi ETL Task support + +Note: This is source-agnostic but not target-agnositc; it has i2b2 + knowledge. + +''' + +from typing import Any, Callable, Dict, Iterator, List, Optional as Opt, Tuple, cast +from contextlib import contextmanager +from datetime import datetime +import csv +import logging + +from luigi.contrib.sqla import SQLAlchemyTarget +from sqlalchemy import text as sql_text, Column, func, MetaData, Table +from sqlalchemy.engine import Connection, Engine +from sqlalchemy.engine.result import ResultProxy +from sqlalchemy.engine.url import make_url +from sqlalchemy.exc import DatabaseError +from sqlalchemy.sql.expression import Select +import sqlalchemy as sqla +import luigi +from cx_Oracle import Error as OraError, _Error as Ora_Error + +from eventlog import EventLogger, LogState, JSONObject # ISSUE: use https://eliot.readthedocs.io/en/1.1.0/ instead? +from param_val import StrParam, IntParam, BoolParam +from script_lib import Script +from sql_syntax import Environment, Params, SQL +from sql_syntax import params_used, insert_append_table + +log = logging.getLogger(__name__) + + +class DBTarget(SQLAlchemyTarget): + '''Take advantage of engine caching logic from SQLAlchemyTarget, + but don't bother with target_table, update_id, etc. + + >>> t = DBTarget(connection_string='sqlite:///') + >>> t.engine.scalar('select 1 + 1') + 2 + ''' + def __init__(self, connection_string: str, + target_table: Opt[str]=None, update_id: Opt[str]=None, + echo: bool=False) -> None: + SQLAlchemyTarget.__init__( + self, connection_string, + target_table=target_table, + update_id=update_id, + echo=echo) + + def exists(self) -> bool: + raise NotImplementedError + + def touch(self) -> None: + raise NotImplementedError + + +class ETLAccount(luigi.Config): + '''Access to connect and run ETL. + + This account needs read access to source material, write access to + the destination i2b2 star schema, and working space for + intermediary tables, views, functions, and such. + ''' + account = StrParam(description='see client.cfg', + default='') + passkey = StrParam(description='see client.cfg', + default='', + significant=False) + ssh_tunnel = StrParam(description='see client.cfg', + default='', + significant=False) + echo = BoolParam(description='SQLAlchemy echo logging', + significant=False) + + +class LoggedConnection(object): + '''Wrap (parts of) the sqlalchemy Connection API with logging. + + If you have an EventLogger, `log`, you can wrap a connection + in a step that explains what the connection is for:: + + with log.step('crunching data') as step: + lc = LoggedConnection(conn, log, step) + + .. note:: A LoggedConnection wraps only the `execute` and `scalar` + methods from the sqlalchemy API. + + ''' + def __init__(self, conn: Connection, log: EventLogger, + step: LogState) -> None: + self._conn = conn + self.log = log + self.step = step + + def __repr__(self) -> str: + return '%s(%s, %s)' % (self.__class__.__name__, self._conn, self.log) + + def _log_args(self, event: str, operation: object, + params: Params) -> Tuple[str, JSONObject, JSONObject]: + msg = '%(event)s %(sql3)s' + ('\n%(params)s' if params else '') + argobj = dict(event=event, sql3=_peek(operation, lines=3), params=params) + extra = dict(statement=str(operation)) + return msg, argobj, extra + + def execute(self, operation: object, params: Opt[Params] = None) -> ResultProxy: + msg, argobj, extra = self._log_args('execute', operation, params or {}) + with self.log.step(msg, argobj, extra): + return self._conn.execute(operation, params or {}) + + def scalar(self, operation: object, params: Opt[Params] = None) -> Any: + msg, argobj, extra = self._log_args('scalar', str(operation), params or {}) + with self.log.step(msg, argobj, extra) as step: + result = self._conn.scalar(operation, params or {}) + step.extra.update(dict(result=result)) + return result + + +def _peek(thing: object, + lines: int=1, + max_len: int=120) -> str: + return '\n'.join(str(thing).split('\n')[:lines])[:180] + + +class DBAccessTask(luigi.Task): + """Manage DB account credentials and logging. + + Typical usage:: + + with self.connection(event='munching data') as conn: + howmany = conn.scalar('select count(*) from cookies') + yummy = conn.execute('select * from cookies') + """ + account = StrParam(default=ETLAccount().account) + ssh_tunnel = StrParam(default=ETLAccount().ssh_tunnel, + significant=False) + passkey = StrParam(default=ETLAccount().passkey, + significant=False) + echo = BoolParam(default=ETLAccount().echo, + significant=False) + max_idle = IntParam(description='Set to less than Oracle profile max idle time.', + default=60 * 20, + significant=False) + _log = logging.getLogger(__name__) # ISSUE: ambient. + + def output(self) -> luigi.Target: + return self._dbtarget() + + def _dbtarget(self) -> DBTarget: + return DBTarget(self._make_url(self.account), + target_table=None, update_id=self.task_id, + echo=self.echo) + + def _make_url(self, account: str) -> str: + url = make_url(account) + if 'oracle' in account.lower(): + url.query['pool_recycle'] = self.max_idle + # `twophase` interferes with direct path load somehow. + # To disable twophase, allow_twophase flag was commented out of opts dict in cx_Oracle.py + # This appears to be a change from cx_Oracle version 5 to 6. + url.query['allow_twophase'] = False + if self.passkey: + from os import environ # ISSUE: ambient + url.password = environ[self.passkey] + if self.ssh_tunnel and url.host: + host, port = self.ssh_tunnel.split(':', 1) + url.host = host + url.port = port + return str(url) + + def log_info(self) -> Dict[str, Any]: + '''Get info to log: luigi params, task_family, task_hash. + ''' + return dict(self.to_str_params(only_significant=True), + task_family=self.task_family, + task_hash=self.task_id[-luigi.task.TASK_ID_TRUNCATE_HASH:]) + + @contextmanager + def connection(self, event: str='connect') -> Iterator[LoggedConnection]: + conn = ConnectionProblem.tryConnect(self._dbtarget().engine) + log = EventLogger(self._log, self.log_info()) + with log.step('%(event)s: <%(account)s>', + dict(event=event, account=self.account)) as step: + yield LoggedConnection(conn, log, step) + + def _fix_password(self, environ: Dict[str, str], getpass: Callable[[str], str]) -> None: + '''for interactive use; e.g. in notebooks + ''' + if self.passkey not in environ: + environ[self.passkey] = getpass(self.passkey) + + +class SqlScriptTask(DBAccessTask): + '''Task to run a stylized SQL script. + + As seen in `script_lib`, a script may require (in the luigi sense) + other scripts and it is complete iff its last query says so. + + Running a script may be parameterized with bind params and/or + Oracle sqlplus style defined `&&variables`: + + >>> variables = dict(I2B2STAR='I2B2DEMODATA') + >>> txform = SqlScriptTask( + ... account='sqlite:///', passkey=None, + ... script=Script.pcornet_loader, + ... param_vars=variables) + + ISSUE: doctest dependencies? + >>> [task.script for task in txform.requires()] + ... #doctest: +ELLIPSIS + [] + + >>> txform.complete() + False + + ''' + script = cast(Script, luigi.EnumParameter(enum=Script)) + param_vars = cast(Environment, luigi.DictParameter(default={})) + _log = logging.getLogger('sql_scripts') # ISSUE: ambient. magic-string + + @property + def variables(self) -> Environment: + '''Defined variables for this task (or task family). + ''' + return self.param_vars + + @property + def vars_for_deps(self) -> Environment: + '''Defined variables to supply to dependencies. + ''' + return self.variables + + def requires(self) -> List[luigi.Task]: + '''Wrap each of `self.script.deps()` in a SqlScriptTask. + ''' + return [type(s.name, (SqlScriptTask,), + { + 'script': s, + 'param_vars': self.vars_for_deps, + 'account': self.account, + 'passkey': self.passkey, + 'echo': self.echo + })() + for s in self.script.deps()] + + def log_info(self) -> Dict[str, Any]: + '''Include script, filename in self.log_info(). + ''' + return dict(DBAccessTask.log_info(self), + script=self.script.name, + filename=self.script.fname) + + def complete(self) -> bool: + '''Each script's last query tells whether it is complete. + + It should be a scalar query that returns non-zero for done + and either zero or an error for not done. + ''' + last_query = self.last_query() + params = params_used(self.complete_params(), last_query) + with self.connection(event=self.task_family + ' complete query: ' + self.script.name) as conn: + try: + result = conn.scalar(sql_text(last_query), params) + return bool(result) + except DatabaseError as exc: + conn.log.warning('%(event)s: %(exc)s', + dict(event='complete query error', exc=exc)) + return False + + def last_query(self) -> SQL: + """ + Note: In order to support run-only variables as in UploadTask, + we skip statements with unbound &&variables. + """ + return self.script.statements( + skip_unbound=True, + variables=self.variables)[-1] + + def complete_params(self) -> Dict[str, Any]: + '''Make `task_id` available to complete query as a bind param. + ''' + return dict(task_id=self.task_id) + + def run(self) -> None: + '''Run each statement in the script without any bind parameters. + ''' + self.run_bound() + + def run_bound(self, + script_params: Opt[Params]=None) -> None: + '''Run with a (default emtpy) set of parameters bound. + ''' + with self.connection(event='run script') as conn: + self.run_event(conn, script_params=script_params) + + def run_event(self, + conn: LoggedConnection, + run_vars: Opt[Environment]=None, + script_params: Opt[Params]=None) -> int: + '''Run script inside a LoggedConnection event. + + @param run_vars: variables to define for this run + @param script_params: parameters to bind for this run + @return: count of rows bulk-inserted + always 0 for this class, but see UploadTask + + To see how a script can ignore errors, see :mod:`script_lib`. + ''' + bulk_rows = 0 + ignore_error = False + run_params = dict(script_params or {}, task_id=self.task_id) + fname = self.script.fname + variables = dict(run_vars or {}, **self.variables) + each_statement = self.script.each_statement(variables=variables) + + for line, _comment, statement in each_statement: + try: + if self.is_bulk(statement): + bulk_rows = self.bulk_insert( + conn, fname, line, statement, run_params, + bulk_rows) + else: + ignore_error = self.execute_statement( + conn, fname, line, statement, run_params, + ignore_error) + except DatabaseError as exc: + db = self._dbtarget().engine + err = SqlScriptError(exc, self.script, line, + statement, str(db)) + if ignore_error: + conn.log.warning('%(event)s: %(error)s', + dict(event='ignore', error=err)) + else: + raise err from None + if bulk_rows > 0: + conn.step.msg_parts.append(' %(rowtotal)s total rows') + conn.step.argobj.update(dict(rowtotal=bulk_rows)) + + return bulk_rows + + def execute_statement(self, conn: LoggedConnection, fname: str, line: int, + statement: SQL, run_params: Params, + ignore_error: bool) -> bool: + '''Log and execute one statement. + ''' + sqlerror = Script.sqlerror(statement) + if sqlerror is not None: + return sqlerror + params = params_used(run_params, statement) + self.set_status_message( + '%s:%s:\n%s\n%s' % (fname, line, statement, params)) + conn.execute(statement, params) + return ignore_error + + def is_bulk(self, statement: SQL) -> bool: + '''always False for this class, but see UploadTask + ''' + return False + + def bulk_insert(self, conn: LoggedConnection, fname: str, line: int, + statement: SQL, run_params: Params, + bulk_rows: int) -> int: + raise NotImplementedError( + 'overriding is_bulk() requires overriding bulk_insert()') + + +class CDMStatusTask(DBAccessTask): + ''' + A DBAccessTask that relies on the CDM status table to assess completion. + + Typical usage is to record the start of the task, run task operations and then record + the end of the task. + self.setTaskStart() + self.load() + self.setTaskEnd(self.getRecordCountFromTable()) + ''' + taskName = StrParam() + + # Basic status check, assume the typical task produces at least one record. + expectedRecords = IntParam(default=1) + + statusTable = Table("cdm_status", MetaData(), + Column('TASK'), Column('START_TIME'), Column('END_TIME'), Column('RECORDS')) + + def complete(self) -> bool: + ''' + Complete when the CDM status table reports at least as many records as expected for the task. + ''' + with self.connection() as q: + statusTableRecordCount = q.scalar('select records from cdm_status where task = \'%s\'' % self.taskName) + + # If true, the task has not been logged in the CDM status table or has been logged and is in an + # inconsistent state with the number of records set to null. + if statusTableRecordCount is None: + return False + + log.info('task %s has %d rows', self.taskName, statusTableRecordCount) + return statusTableRecordCount >= self.expectedRecords # type: ignore # sqla + + def getRecordCountFromTable(self) -> int: + ''' + Queries the database for the number of records in the table named taskName + ''' + # This op is out of sync with the rest of the class, in that it assumes + # the task must represent the creation of a table in the db. + with self.connection() as q: + return q.scalar(sqla.select([func.count()]).select_from(self.taskName)) # type: ignore + + def setTaskEnd(self, rowCount: int) -> None: + ''' + Updates the taskName entry in the CDM status table with an end time of now and a count of records. + ''' + st = self.statusTable + db = self._dbtarget().engine + db.execute(st.update().where(st.c.TASK == self.taskName), + [{'END_TIME': datetime.now(), 'RECORDS': rowCount}]) + + def setTaskStart(self) -> None: + ''' + Adds taskName to the CDM status table with a start time of now. + ''' + st = self.statusTable + db = self._dbtarget().engine + # prune any failed attempt + db.execute(st.delete().where(sqla.and_(st.c.TASK == self.taskName, + st.c.END_TIME == None))) # noqa + db.execute(st.insert(), [{'TASK': self.taskName, 'START_TIME': datetime.now()}]) + + +def log_plan(lc: LoggedConnection, event: str, params: Dict[str, Any], + query: Opt[Select]=None, sql: Opt[str]=None) -> None: + if query is not None: + sql = str(query.compile(bind=lc._conn)) + if sql is None: + return + plan = explain_plan(lc, sql) + param_msg = ', '.join('%%(%s)s' % k for k in params.keys()) + lc.log.info('%(event)s [' + param_msg + ']\n' + 'query: %(query_peek)s plan:\n' + '%(plan)s', + dict(params, event=event, query_peek=_peek(sql), + plan='\n'.join(plan))) + + +def explain_plan(work: LoggedConnection, statement: SQL) -> List[str]: + work.execute('explain plan for ' + statement) + # ref 19 Using EXPLAIN PLAN + # Oracle 10g Database Performance Tuning Guide + # https://docs.oracle.com/cd/B19306_01/server.102/b14211/ex_plan.htm + plan = work.execute( + 'SELECT PLAN_TABLE_OUTPUT line FROM TABLE(DBMS_XPLAN.DISPLAY())') + return [row.line for row in plan] + + +def maybe_ora_err(exc: Exception) -> Opt[Ora_Error]: + if isinstance(exc, DatabaseError): + if isinstance(exc.orig, OraError): + return cast(Ora_Error, exc.orig.args[0]) + return None + + +class SqlScriptError(IOError): + '''Include script file, line number in diagnostics + ''' + def __init__(self, exc: Exception, script: Script, line: int, statement: SQL, + conn_label: str) -> None: + fname = script.name + message = '%s <%s>\n%s:%s:\n' + args = [exc, conn_label, fname, line] + ora_ex = maybe_ora_err(exc) + if ora_ex: + offset = ora_ex.offset + message += '%s%s' + args[0] = ora_ex.message + args += [_pick_lines(statement[:offset], -3, None), + _pick_lines(statement[offset:], None, 3)] + else: + message += '%s' + args += [statement] + + self.message = message + self.args = tuple(args) + + def __str__(self) -> str: + return self.message % self.args + + +def _pick_lines(s: str, lo: Opt[int], hi: Opt[int]) -> str: + return '\n'.join(s.split('\n')[lo:hi]) + + +class TimeStampParameter(luigi.Parameter): + '''A datetime interchanged as milliseconds since the epoch. + ''' + + def parse(self, s: str) -> datetime: + ms = int(s) + return datetime.fromtimestamp(ms / 1000.0) + + def serialize(self, dt: datetime) -> str: + epoch = datetime.utcfromtimestamp(0) + ms = (dt - epoch).total_seconds() * 1000 + return str(int(ms)) + + +class SourceTask(luigi.Task): + @property + def source_cd(self) -> str: + raise NotImplementedError + + @property + def download_date(self) -> datetime: + raise NotImplementedError + + +class I2B2Task(object): + @property + def project(self) -> 'I2B2ProjectCreate': + return I2B2ProjectCreate() + + +class UploadTask(I2B2Task, SqlScriptTask): + '''Run a script with an associated `upload_status` record. + ''' + @property + def source(self) -> SourceTask: + raise NotImplementedError('subclass must implement') + + @property + def transform_name(self) -> str: + return self.task_id + + def complete_params(self) -> Dict[str, Any]: + return dict(task_id=self.task_id, + download_date=self.source.download_date) + + def output(self) -> luigi.Target: + return self._upload_target() + + def _upload_target(self) -> 'UploadTarget': + return UploadTarget(self._make_url(self.account), + self.project.upload_table, + self.transform_name, self.source, + echo=self.echo) + + def requires(self) -> List[luigi.Task]: + return [self.project, self.source] + SqlScriptTask.requires(self) + + def complete(self) -> bool: + # Belt and suspenders + return (self.output().exists() and + SqlScriptTask.complete(self)) + + @property + def label(self) -> str: + return self.script.title + + def run(self) -> None: + upload = self._upload_target() + with upload.job(self, + label=self.label, + user_id=make_url(self.account).username) as conn_id_r: + conn, upload_id, result = conn_id_r + bulk_rows = SqlScriptTask.run_event( + self, conn, + run_vars=dict(upload_id=str(upload_id)), + script_params=dict(self.script_params(), upload_id=upload_id)) + result[upload.table.c.loaded_record.name] = bulk_rows + + def script_params(self) -> Params: + return dict(download_date=self.source.download_date, + project_id=self.project.project_id) + + def is_bulk(self, statement: SQL) -> bool: + return insert_append_table(statement) is not None + + def bulk_insert(self, conn: LoggedConnection, fname: str, line: int, + statement: SQL, run_params: Params, + bulk_rows: int) -> int: + with conn.log.step( + '%(filename)s:%(lineno)s: %(event)s', + dict(event='bulk_insert', + filename=fname, lineno=line)) as step: + plan = '\n'.join(explain_plan(conn, statement)) + conn.log.info('%(filename)s:%(lineno)s: %(event)s:\n%(plan)s', + dict(filename=fname, lineno=line, event='plan', + plan=plan)) + + params = params_used(run_params, statement) + self.set_status_message( + '%s:%s:\n%s\n%s' % (fname, line, statement, params)) + last_result = conn.execute(statement, params) + step.msg_parts.append(' %(rowcount)d rows') + step.argobj.update(dict(rowcount=last_result.rowcount)) + bulk_rows += last_result.rowcount + + return bulk_rows + + +class UploadTarget(DBTarget): + def __init__(self, connection_string: str, + table: sqla.Table, transform_name: str, source: SourceTask, + echo: bool=False) -> None: + DBTarget.__init__(self, connection_string, + echo=echo) + self.table = table + self.source = source + self.transform_name = transform_name + self.upload_id = None # type: Opt[int] + + def __repr__(self) -> str: + return '%s(transform_name=%s)' % ( + self.__class__.__name__, self.transform_name) + + def exists(self) -> bool: + conn = ConnectionProblem.tryConnect(self.engine) + with conn.begin(): + up_t = self.table + upload_id = conn.scalar( + sqla.select([sqla.func.max(up_t.c.upload_id)]) + .select_from(up_t) + .where(sqla.and_(up_t.c.transform_name == self.transform_name, + up_t.c.load_status == 'OK'))) + return upload_id is not None + + @contextmanager + def job(self, task: DBAccessTask, + label: Opt[str] = None, user_id: Opt[str] = None, + upload_id: Opt[int] = None) -> Iterator[ + Tuple[LoggedConnection, int, Params]]: + event = 'upload job' + with task.connection(event=event) as conn: + up_t = self.table + if upload_id is None: + if user_id is None: + raise TypeError('must supply user_id for new record') + if label is None: + raise TypeError('must supply label for new record') + upload_id = self.insert(conn, label, user_id) + else: + [label, user_id] = conn.execute( + sqla.select([up_t.c.upload_label, up_t.c.user_id]) + .where(up_t.c.upload_id == upload_id)).fetchone() + + msg = ' %(upload_id)s for %(label)s' + info = dict(label=label, upload_id=upload_id) + conn.step.msg_parts.append(msg) + conn.step.argobj.update(info) + conn.log.info(msg, info) # Go ahead and log the upload_id early. + + result = {} # type: Params + yield conn, upload_id, result + conn.execute(up_t.update() + .where(up_t.c.upload_id == upload_id) + .values(load_status='OK', end_date=func.now(), + **result)) + + def insert(self, conn: LoggedConnection, label: str, user_id: str) -> int: + ''' + :param label: a label for related facts for audit purposes + :param user_id: an indication of who uploaded the related facts + ''' + up_t = self.table + next_q = sql_text( + '''select {i2b2}.sq_uploadstatus_uploadid.nextval + from dual'''.format(i2b2=self.table.schema)) + upload_id = conn.scalar(next_q) # type: int + + conn.execute(up_t.insert() + .values(upload_id=upload_id, + upload_label=label, + user_id=user_id, + source_cd=self.source.source_cd, + load_date=sqla.func.now(), + transform_name=self.transform_name)) + self.upload_id = upload_id + return upload_id + + +class I2B2ProjectCreate(DBAccessTask): + star_schema = StrParam(description='see client.cfg') + project_id = StrParam(description='see client.cfg') + _meta = None # type: Opt[sqla.MetaData] + _upload_table = None # type: Opt[sqla.Table] + + Column, ty = sqla.Column, sqla.types + upload_status_columns = [ + Column('upload_id', ty.Numeric(38, 0, asdecimal=False), primary_key=True), + Column('upload_label', ty.String(500), nullable=False), + Column('user_id', ty.String(100), nullable=False), + Column('source_cd', ty.String(50), nullable=False), + Column('no_of_record', ty.Numeric(asdecimal=False)), + Column('loaded_record', ty.Numeric(asdecimal=False)), + Column('deleted_record', ty.Numeric(asdecimal=False)), + Column('load_date', ty.DateTime, nullable=False), + Column('end_date', ty.DateTime), + Column('load_status', ty.String(100)), + Column('message', ty.Text), + Column('input_file_name', ty.Text), + Column('log_file_name', ty.Text), + Column('transform_name', ty.String(500)), + ] + + def output(self) -> 'SchemaTarget': + return SchemaTarget(self._make_url(self.account), + schema_name=self.star_schema, + table_eg='patient_dimension', + echo=self.echo) + + def run(self) -> None: + raise NotImplementedError('see heron_create.create_deid_datamart etc.') + + @property + def metadata(self) -> sqla.MetaData: + if self._meta: + return self._meta + self._meta = meta = sqla.MetaData(schema=self.star_schema) + return meta + + def table_details(self, lc: LoggedConnection, tables: List[str]) -> sqla.MetaData: + i2b2_meta = sqla.MetaData(schema=self.star_schema) + i2b2_meta.reflect(only=tables, schema=self.star_schema, + bind=self._dbtarget().engine) + return i2b2_meta + + @property + def upload_table(self) -> sqla.Table: + if self._upload_table is not None: + return self._upload_table + t = sqla.Table( + 'upload_status', self.metadata, + *self.upload_status_columns, + schema=self.star_schema) + self._upload_table = t + return t + + +class SchemaTarget(DBTarget): + def __init__(self, connection_string: str, schema_name: str, table_eg: str, + echo: bool=False) -> None: + DBTarget.__init__(self, connection_string, echo=echo) + self.schema_name = schema_name + self.table_eg = table_eg + + def exists(self) -> bool: + table = Table(self.table_eg, sqla.MetaData(), schema=self.schema_name) + return table.exists(bind=self.engine) # type: ignore + + +class ConnectionProblem(DatabaseError): + '''Provide hints about ssh tunnels. + ''' + # connection closed, no listener + tunnel_hint_codes = [12537, 12541] + + @classmethod + def tryConnect(cls, engine: Engine) -> Connection: + try: + return engine.connect() + except DatabaseError as exc: + raise ConnectionProblem.refine(exc, str(engine)) from None + + @classmethod + def refine(cls, exc: DatabaseError, conn_label: str) -> Exception: + '''Recognize known connection problems. + + :returns: customized exception for known + problem else exc + ''' + ora_ex = maybe_ora_err(exc) + + if ora_ex: + return cls(exc, ora_ex, conn_label) + return exc + + def __init__(self, exc: DatabaseError, ora_ex: Ora_Error, conn_label: str) -> None: + DatabaseError.__init__( + self, + exc.statement, exc.params, + exc.connection_invalidated) + message = '%s <%s>' + args = [ora_ex, conn_label] + + if exc.statement and ora_ex.offset: + stmt_rest = exc.statement[ + ora_ex.offset:ora_ex.offset + 120] + message += '\nat: %s' + args += [stmt_rest] + local_conn_prob = ( + ora_ex.code in self.tunnel_hint_codes and + 'localhost' in conn_label) + if local_conn_prob: + message += '\nhint: ssh tunnel down?' + message += '\nin: %s' + args += [ora_ex.context] + self.message = message + self.args = tuple(args) + + def __str__(self) -> str: + return self.message % self.args + + +class ReportTask(DBAccessTask): + @property + def script(self) -> Script: + raise NotImplementedError('subclass must implement') + + @property + def report_name(self) -> str: + raise NotImplementedError('subclass must implement') + + def complete(self) -> bool: + '''Double-check requirements as well as output. + ''' + deps = luigi.task.flatten(self.requires()) # type: List[luigi.Task] + return (self.output().exists() and + all(t.complete() for t in deps)) + + def _csvout(self) -> 'CSVTarget': + return CSVTarget(path=self.report_name + '.csv') + + def output(self) -> luigi.Target: + return self._csvout() + + def run(self) -> None: + with self.connection('report') as conn: + query = sql_text( + 'select * from {object}'.format(object=self.report_name)) + result = conn.execute(query) + cols = result.keys() + rows = result.fetchall() + self._csvout().export(cols, rows) + + +class CSVTarget(luigi.local_target.LocalTarget): + def export(self, cols: List[str], data: List) -> None: + with self.open('wb') as stream: + dest = csv.writer(stream) + dest.writerow(cols) + dest.writerows(data) + + @contextmanager + def dictreader(self, + lowercase_fieldnames: bool=False, + delimiter: str=',') -> Iterator[csv.DictReader]: + '''DictReader contextmanager + + @param lowercase_fieldnames: sqlalchemy uses lower-case bind + parameter names, but SCILHS CSV file headers use the + actual uppercase column names. So we got: + + CompileError: The 'oracle' dialect with current + database version settings does not support empty + inserts. + + + ''' + with self.open('rb') as stream: + dr = csv.DictReader(stream, delimiter=delimiter) + if lowercase_fieldnames: + # This is a bit of a kludge, but it works... + dr.fieldnames = [n.lower() for n in dr.fieldnames] + yield dr + + +class AdHoc(DBAccessTask): + sql = StrParam() + name = StrParam() + + def _csvout(self) -> CSVTarget: + return CSVTarget(path=self.name + '.csv') + + def output(self) -> luigi.Target: + return self._csvout() + + def run(self) -> None: + with self.connection() as work: + result = work.execute(self.sql) + cols = result.keys() + rows = result.fetchall() + self._csvout().export(cols, rows) + + +class KillSessions(DBAccessTask): + reason = StrParam(default='*no reason given*') + sql = ''' + begin + sys.kill_own_sessions(:reason); + end; + ''' + + def complete(self) -> bool: + return False + + def run(self) -> None: + with self.connection('kill own sessions') as work: + work.execute(self.sql, params=dict(reason=self.reason)) + + +class AlterStarNoLogging(DBAccessTask): + sql = ''' + alter table TABLE nologging + ''' + tables = ['patient_mapping', + 'encounter_mapping', + 'patient_dimension', + 'visit_dimension', + 'observation_fact'] + + def complete(self) -> bool: + return False + + def run(self) -> None: + with self.connection() as work: + for table in self.tables: + work.execute(self.sql.replace('TABLE', table)) + + +class MigrateUpload(SqlScriptTask, I2B2Task): + upload_id = IntParam() + workspace_star = StrParam() + parallel_degree = IntParam(default=24, + significant=False) + + script = Script.pcornet_loader + + @property + def variables(self) -> Environment: + return dict(I2B2STAR=self.project.star_schema, + workspace_star=self.workspace_star, + parallel_degree=str(self.parallel_degree), + upload_id=str(self.upload_id)) + + +class MigratePendingUploads(DBAccessTask, I2B2Task, luigi.WrapperTask): + workspace_star = StrParam() + + find_pending = """ + select upload_id from %(WORKSPACE)s.upload_status + where load_status in ('OK', 'OK_work') and upload_id not in ( + select upload_id from %(I2B2STAR)s.upload_status + where load_status='OK' ) + """ + + def requires(self) -> List[luigi.Task]: + find_pending = self.find_pending % dict( + WORKSPACE=self.workspace_star, + I2B2STAR=self.project.star_schema) + + deps = [] # type: List[luigi.Task] + with self.connection('pending uploads') as lc: + pending = [row.upload_id for row in + lc.execute(find_pending).fetchall()] + + workmeta = sqla.MetaData() + for upload_id in pending: + table = Table('observation_fact_%d' % upload_id, workmeta, + schema=self.workspace_star) + if table.exists(bind=lc._conn): + deps.append( + MigrateUpload(upload_id=upload_id, + workspace_star=self.workspace_star)) + else: + log.warn('no such table to migrate: %s', table) + return deps diff --git a/eventlog.py b/eventlog.py new file mode 100644 index 0000000..3067cf3 --- /dev/null +++ b/eventlog.py @@ -0,0 +1,153 @@ +'''eventlog -- structured logging support + +EventLogger for nested events +============================= + +Suppose we want to log some events which naturally nest. + +First, add a handler to a logger: + +>>> import logging, sys +>>> log1 = logging.getLogger('log1') +>>> detail = logging.StreamHandler(sys.stdout) +>>> log1.addHandler(detail) +>>> log1.setLevel(logging.INFO) +>>> detail.setFormatter(logging.Formatter( +... fmt='%(levelname)s %(elapsed)s %(do)s %(message)s')) + +>>> io = MockIO() +>>> event0 = EventLogger(log1, dict(customer='Jones', invoice=123), io.clock) + + +>>> with event0.step('Build %(product)s', dict(product='house')): +... with event0.step('lay foundation %(depth)d ft deep', +... dict(depth=20)) as info: +... info.msg_parts.append(' at %(temp)d degrees') +... info.argobj['temp'] = 65 +... with event0.step('frame %(stories)d story house', +... dict(stories=2)) as info: +... pass +... start, elapsed, _ms = event0.elapsed() +... eta = event0.eta(pct=25) +... # doctest: +ELLIPSIS +INFO ('... 12:30:01', None, None) begin 0:00:00 [1] Build house... +INFO ('... 12:30:03', None, None) begin 0:00:02 [1, 2] lay foundation 20 ft deep... +INFO ('... 12:30:03', '0:00:03', 3000000) end 0:00:03 [1, 2] lay foundation 20 ft deep at 65 degrees. +INFO ('... 12:30:10', None, None) begin 0:00:09 [1, 3] frame 2 story house... +INFO ('... 12:30:10', '0:00:05', 5000000) end 0:00:05 [1, 3] frame 2 story house. +INFO ('... 12:30:01', '0:00:35', 35000000) end 0:00:35 [1] Build house. + +>>> eta +datetime.datetime(2000, 1, 1, 12, 31, 49) + +''' + +from contextlib import contextmanager +from datetime import datetime +from typing import ( + Any, Callable, Dict, Iterator, List, MutableMapping, + NamedTuple, Optional as Opt, TextIO, Tuple +) +import logging + + +KWArgs = MutableMapping[str, Any] +JSONObject = Dict[str, Any] + +LogState = NamedTuple('LogState', [ + ('msg_parts', List[str]), + ('argobj', JSONObject), + ('extra', JSONObject)]) + + +class EventLogger(logging.LoggerAdapter): + def __init__(self, logger: logging.Logger, event: JSONObject, + clock: Opt[Callable[[], datetime]]=None) -> None: + logging.LoggerAdapter.__init__(self, logger, extra={}) +# self.name = logger.name + if clock is None: + clock = datetime.now # ISSUE: ambient + self.event = event + self._clock = clock + self._seq = 0 + self._step = [] # type: List[Tuple[int, datetime]] + List # let flake8 know we're using it + + def __repr__(self) -> str: + # typeshed doesn't know that LoggerAdapter has .name? + return '%s(%s, %s)' % (self.__class__.__name__, self.name, self.event) # type: ignore + + def process(self, msg: str, kwargs: KWArgs) -> Tuple[str, KWArgs]: + extra = dict(kwargs.get('extra', {}), + context=self.event) + return msg, dict(kwargs, extra=extra) + + def elapsed(self, then: Opt[datetime]=None) -> Tuple[str, str, int]: + start = then or self._step[-1][1] + elapsed = self._clock() - start + ms = int(elapsed.total_seconds() * 1000000) + return (str(start), str(elapsed), ms) + + def eta(self, pct: float) -> datetime: + t0 = self._step[0][1] + elapsed = self._clock() - t0 + return t0 + elapsed * (1 / (pct / 100)) + + @contextmanager + def step(self, msg: str, argobj: Dict[str, object], + extra: Opt[Dict[str, object]]=None) -> Iterator[LogState]: + checkpoint = self._clock() + self._seq += 1 + self._step.append((self._seq, checkpoint)) + extra = extra or {} + fmt_step = '%(t_step)s %(step)s ' + step_ixs = [ix for (ix, _t) in self._step] + t_step = str(checkpoint - self._step[0][1]) + self.info(fmt_step + msg + '...', + dict(argobj, step=step_ixs, t_step=t_step), + extra=dict(extra, do='begin', + elapsed=(str(checkpoint), None, None))) + msgparts = [msg] + outcome = logging.INFO + try: + yield LogState(msgparts, argobj, extra) + except: # noqa + outcome = logging.ERROR + raise + finally: + elapsed = self.elapsed(then=checkpoint) + self.log(outcome, ''.join([fmt_step] + msgparts) + '.', + dict(argobj, step=step_ixs, t_step=elapsed[1]), + extra=dict(extra, do='end', + elapsed=elapsed)) + self._step.pop() + + +class TextFilter(logging.Filter): + def __init__(self, skips: List[str]) -> None: + self.skips = skips + + def filter(self, record: logging.LogRecord) -> bool: + for skip in self.skips: + if record.getMessage().startswith(skip): + return False + return True + + +class TextHandler(logging.StreamHandler): + def __init__(self, stream: TextIO, skips: List[str]=[]) -> None: + logging.StreamHandler.__init__(self, stream) + self.addFilter(TextFilter(skips)) + + +class MockIO(object): + def __init__(self, + now: datetime=datetime(2000, 1, 1, 12, 30, 0)) -> None: + self._now = now + self._delta = 1 + + def clock(self) -> datetime: + from datetime import timedelta + self._now += timedelta(seconds=self._delta) + self._delta += 1 + return self._now diff --git a/i2p_tasks.py b/i2p_tasks.py new file mode 100644 index 0000000..56b4cbe --- /dev/null +++ b/i2p_tasks.py @@ -0,0 +1,467 @@ +"""i2p_tasks -- Luigi CDM task support. +""" +from typing import cast, List, Type + +import luigi +from sqlalchemy.engine import RowProxy +from sqlalchemy.exc import DatabaseError + +from csv_load import LoadCSV +from etl_tasks import CDMStatusTask, SqlScriptTask +from param_val import IntParam, StrParam +from script_lib import Script +from sql_syntax import Environment, Params + +import csv +import subprocess +import urllib.request + + +class I2PConfig(luigi.Config): + datamart_id = StrParam(description='see client.cfg') + datamart_name = StrParam(description='see client.cfg') + enrollment_months_back = StrParam(description='see client.cfg') + i2b2_data_schema = StrParam(description='see client.cfg') + i2b2_etl_schema = StrParam(description='see client.cfg') + i2b2_meta_schema = StrParam(description='see client.cfg') + min_pat_list_date_dd_mon_rrrr = StrParam(description='see client.cfg') + min_visit_date_dd_mon_rrrr = StrParam(description='see client.cfg') + network_id = StrParam(description='see client.cfg') + network_name = StrParam(description='see client.cfg') + + +class I2PScriptTask(SqlScriptTask): + + @property + def variables(self) -> Environment: + return dict(datamart_id=I2PConfig().datamart_id, datamart_name=I2PConfig().datamart_name, + i2b2_data_schema=I2PConfig().i2b2_data_schema, + min_pat_list_date_dd_mon_rrrr=I2PConfig().min_pat_list_date_dd_mon_rrrr, + min_visit_date_dd_mon_rrrr=I2PConfig().min_visit_date_dd_mon_rrrr, + i2b2_meta_schema=I2PConfig().i2b2_meta_schema, + enrollment_months_back=I2PConfig().enrollment_months_back, network_id=I2PConfig().network_id, + network_name=I2PConfig().network_name, i2b2_etl_schema=I2PConfig().i2b2_etl_schema) + + +class condition(I2PScriptTask): + script = Script.condition + + def requires(self) -> List[luigi.Task]: + return [encounter()] + + +class death(I2PScriptTask): + script = Script.death + + def requires(self) -> List[luigi.Task]: + return [demographic()] + + +class death_cause(I2PScriptTask): + script = Script.death_cause + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class demographic(I2PScriptTask): + script = Script.demographic + + def requires(self) -> List[luigi.Task]: + return [loadLanguage(), pcornet_init()] + + +class diagnosis(I2PScriptTask): + script = Script.diagnosis + + def requires(self) -> List[luigi.Task]: + return [encounter()] + + +class dispensing(I2PScriptTask): + script = Script.dispensing + + def requires(self) -> List[luigi.Task]: + return [encounter(), loadRouteMap(), loadUnitMap()] + + +class encounter(I2PScriptTask): + script = Script.encounter + + def requires(self) -> List[luigi.Task]: + return [demographic(), loadPayerMap()] + + +class enrollment(I2PScriptTask): + script = Script.enrollment + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class harvest(I2PScriptTask): + script = Script.harvest + + def requires(self) -> List[luigi.Task]: + return [condition(), death(), death_cause(), diagnosis(), dispensing(), enrollment(), + lab_result_cm(), loadHarvestLocal(), med_admin(), obs_clin(), obs_gen(), pcornet_trial(), + prescribing(), pro_cm(), procedures(), provider(), vital()] + + +class lab_result_cm(I2PScriptTask): + script = Script.lab_result_cm + + def requires(self) -> List[luigi.Task]: + return [encounter(), loadLabNormal(), loadSpecimenSourceMap()] + + +class med_admin(I2PScriptTask): + script = Script.med_admin + + def requires(self) -> List[luigi.Task]: + return [encounter(), loadRouteMap(), loadUnitMap()] + + +class obs_clin(I2PScriptTask): + script = Script.obs_clin + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class obs_gen(I2PScriptTask): + script = Script.obs_gen + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class I2PPatientGroupTask(I2PScriptTask): + patient_num_first = IntParam() + patient_num_last = IntParam() + patient_num_qty = IntParam(significant=False, default=-1) + group_num = IntParam(significant=False, default=-1) + group_qty = IntParam(significant=False, default=-1) + + def run(self) -> None: + SqlScriptTask.run_bound(self, script_params=dict( + patient_num_first=self.patient_num_first, patient_num_last=self.patient_num_last)) + + +class _PatientNumGrouped(luigi.WrapperTask): + group_tasks = cast(List[Type[I2PPatientGroupTask]], []) # abstract + + def requires(self) -> List[luigi.Task]: + deps = [] # type: List[luigi.Task] + for group_task in self.group_tasks: + survey = patient_chunks_survey() + deps += [survey] + results = survey.results() + if results: + deps += [ + group_task( + group_num=ntile.chunk_num, + group_qty=len(results), + patient_num_qty=ntile.patient_num_qty, + patient_num_first=ntile.patient_num_first, + patient_num_last=ntile.patient_num_last) + for ntile in results + ] + return deps + + +class patient_chunks_survey(SqlScriptTask): + script = Script.patient_chunks_survey + patient_chunks = IntParam(default=20) + patient_chunk_max = IntParam(default=None) + + @property + def variables(self) -> Environment: + return dict(chunk_qty=str(self.patient_chunks)) + + def run(self) -> None: + SqlScriptTask.run_bound(self, script_params=dict(chunk_qty=str(self.patient_chunks))) + + def results(self) -> List[RowProxy]: + with self.connection(event='survey results') as lc: + q = ''' + select patient_num + , patient_num_qty + , patient_num_first + , patient_num_last + from patient_chunks + where chunk_qty = :chunk_qty + and (:chunk_max is null or + chunk_num <= :chunk_max) + order by chunk_num + ''' + Params + params = dict(chunk_max=self.patient_chunk_max, chunk_qty=self.patient_chunks) # type: Params + + try: + return lc.execute(q, params=params).fetchall() + except DatabaseError: + return [] + + +# TODO: pcornet_init drops and recreates the cdm_status table, forcing all tasks to wait until init is done. +# Moving this operation to a distinct task would allow some other tasks (e.g. mapping tasks) to proceed, while init +# performs other labor intensive SQL operations. +# In the mean time, don't forget to make all tasks that use the status table dependent on pcornet_init. +# TODO: On a related matter, if the cdm_status table is missing (e.g. on a db where CDM has never been run), running +# the full pipeline, starting at pcornet_loader, will fail. It would be nice to detect this situation and first +# build the status table before attempting other tasks. +class pcornet_init(I2PScriptTask): + script = Script.pcornet_init + + def requires(self) -> List[luigi.Task]: + return [] + + +class pcornet_loader(I2PScriptTask): + script = Script.pcornet_loader + + def requires(self) -> List[luigi.Task]: + return [pcornet_init(), harvest()] + + +class pcornet_trial(I2PScriptTask): + script = Script.pcornet_trial + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class prescribing(I2PScriptTask): + script = Script.prescribing + + def requires(self) -> List[luigi.Task]: + return [encounter(), loadRouteMap(), loadUnitMap()] + + +class pro_cm(I2PScriptTask): + script = Script.pro_cm + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class procedures(I2PScriptTask): + script = Script.procedures + + def requires(self) -> List[luigi.Task]: + return [encounter()] + + +class provider(I2PScriptTask): + script = Script.provider + + def requires(self) -> List[luigi.Task]: + return [loadSpecialtyCode(), encounter()] + + +class vital(I2PScriptTask): + script = Script.vital + + def requires(self) -> List[luigi.Task]: + return [encounter()] + + +class loadPayerMap(LoadCSV): + taskName = 'PAYER_MAP' + # payer_map.csv is a combination of the CDM spec's payer_type spreadsheet + # with payer_name and financial_class values from epic's clarity_epm table. + # payer_name and financial_class are used together to determine the CDM's + # payer type code. + # TODO: incorporte IDX. + csvname = 'curated_data/payer_map.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class loadLabNormal(LoadCSV): + taskName = 'LABNORMAL' + csvname = 'curated_data/labnormal.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class loadLabRUnit(LoadCSV): + taskName = 'LABRUNIT' + csvname = 'curated_data/resultunit_manualcuration.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class loadHarvestLocal(LoadCSV): + taskName = 'HARVEST_LOCAL' + csvname = 'curated_data/harvest_local.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class loadLanguage(LoadCSV): + taskName = 'LANGUAGE_CODE' + # language.csv is a copy of the CDM spec's patient_pref_language_spoken spreadsheet. + # It maps a language code to descriptive text. When the spreadsheet mapped more than + # one text value to a code, duplicate codes where created in language.csv. + csvname = 'curated_data/language.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class loadSpecimenSourceMap(LoadCSV): + taskName = 'SPECIMEN_SOURCE_MAP' + # specimen_source_map.csv matches values in the CDM spec's specimen_source spreadsheet + # to specimen type values from Epic's order_proc table. + csvname = 'curated_data/specimen_source_map.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class loadRouteMap(LoadCSV): + taskName = 'ROUTE_MAP' + # route_map.csv matches values in the CDM spec's _route spreadsheet + # to route values from Epic's zc_admin_route table. + csvname = 'curated_data/route_map.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class loadUnitMap(LoadCSV): + taskName = 'UNIT_MAP' + # unit_map.csv matches values in the CDM spec's _unit spreadsheet + # to unit values from Epic's zc_med_unit table. + csvname = 'curated_data/unit_map.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class NPIDownloadConfig(luigi.Config): + # The configured 'path' and 'npi' variables are used by the downloadNPI method to fetch and + # store the NPPES zip file. Changes to these may require changes to the file system. + # TODO: Update code to discover the npi_csv automatically. + dl_path = StrParam(description='Path where the NPPES zip file will be stored and unzipped.') + extract_path = StrParam(description='Path where the extract') + npi_csv = StrParam(description='CSV file in the NPPES zip that contains NPI data.') + npi_url = StrParam(description='URL for the NPPES download site.') + npi_zip = StrParam(description='Name of the NPPES zip file.') + + # The configured 'col' and 'ct' variables reflect the layout of the NPI data file. + # The extracNPI method uses these values to parse the NPI data file. + # Changes to these may require code changes. + # Complete overkill making these configurable. Consider reverting to hard coded values. + taxonomy_col = StrParam(description='Header for the taxonomy columns in the NPI data file.') + switch_col = StrParam(description='Header for the switch columns in the NPI data file.') + npi_col = StrParam(description='Header for the NPI column in the NPI data file.') + taxonomy_ct = IntParam(description='Number of taxonomy columns in the NPI data file.') + + +class loadSpecialtyMap(LoadCSV): + taskName = 'PROVIDER_SPECIALTY_MAP' + # provider_specialty_map.csv is created on demand by the extractNPI method. + # It maps a National Provider Identifier (NPI) to a primary provider specialty code. + csvname = 'curated_data/provider_specialty_map.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init(), extractNPI()] + + +class loadSpecialtyCode(LoadCSV): + taskName = 'PROVIDER_SPECIALTY_CODE' + # provider_specialty_code.csv is a copy of the CDM spec's provider_primary_specialty spreadsheet. + # It maps a provider specialty code to a descriptive text and grouping. + csvname = 'curated_data/provider_specialty_code.csv' + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class downloadNPI(CDMStatusTask): + ''' + Download the NPPES zip file and extract the NPI data file. + ''' + taskName = 'NPI_DOWNLOAD' + expectedRecords = 0 + + dl_path = NPIDownloadConfig().dl_path + npi_url = NPIDownloadConfig().npi_url + npi_zip = NPIDownloadConfig().npi_zip + + def run(self) -> None: + self.setTaskStart() + self.fetch() + self.unzip() + self.setTaskEnd(self.expectedRecords) + + def fetch(self) -> None: + r = urllib.request.urlopen(self.npi_url + self.npi_zip) + + with open(self.dl_path + self.npi_zip, 'wb') as fout: + fout.write(r.read()) + + def unzip(self) -> None: + subprocess.call(['unzip', '-o', self.dl_path + self.npi_zip, '-d', self.dl_path]) # ISSUE: ambient + + def requires(self) -> List[luigi.Task]: + return [pcornet_init()] + + +class extractNPI(CDMStatusTask): + ''' + Extract the Nation Provider Identifier (NPI) and primary specialty from the NPPES download file + and write the data to a separate csv file. + ''' + taskName = 'NPI_EXTRACT' + specialty_csv = 'provider_specialty_map.csv' + + dl_path = NPIDownloadConfig().dl_path + extract_path = NPIDownloadConfig().extract_path + npi_col = NPIDownloadConfig().npi_col + npi_csv = NPIDownloadConfig().npi_csv + switch_col = NPIDownloadConfig().switch_col + taxonomy_col = NPIDownloadConfig().taxonomy_col + + taxonomy_ct = NPIDownloadConfig().taxonomy_ct + + def run(self) -> None: + self.setTaskStart() + self.extract() + self.setTaskEnd(self.expectedRecords) + + def requires(self) -> List[luigi.Task]: + return [downloadNPI()] + + def extract(self) -> None: + + self.expectedRecords = 0 + + with open(self.dl_path + self.npi_csv, 'r', encoding='utf-8') as fin: + with open(self.extract_path + self.specialty_csv, 'w', newline='') as fout: + reader = csv.DictReader(fin) + writer = csv.writer(fout) + writer.writerow(['NPI', 'SPECIALTY']) + for row in reader: + self.expectedRecords = self.expectedRecords + 1 + useDefault = True + # Search the taxonomy columns for a primary specialty. + for i in range(1, self.taxonomy_ct + 1): + # 'Y' in the switch column indicates that the current taxonomy column contains + # the primary specialty. + if row[self.switch_col + str(i)] == 'Y': + useDefault = False + writer.writerow([row[self.npi_col], row[self.taxonomy_col + str(i)]]) + continue + + # If true, the NPI file did not explicitly identify a primary specialty. Use the value + # in taxonomy column one as the default. + if (useDefault): + writer.writerow([row[self.npi_col], row[self.taxonomy_col + str(1)]]) diff --git a/logging.cfg b/logging.cfg new file mode 100644 index 0000000..6d305bb --- /dev/null +++ b/logging.cfg @@ -0,0 +1,71 @@ +# see section 15.8.3. Configuration file format in +# http://docs.python.org/2/library/logging.config.html +# +# also: +# https://github.com/pysysops/docker-luigi-taskrunner/blob/master/etc/luigi/logging.cfg +# https://pypi.python.org/pypi/python-json-logger +# https://stedolan.github.io/jq/manual/v1.5/ +# + + +[loggers] +keys=root,luigi_debug + +[logger_root] +level=INFO +handlers=console,detail +#handlers=console + +[logger_luigi_debug] +level=DEBUG +handlers=luigi_debug +qualname=luigi-interface + +[handlers] +keys=console,detail,luigi_debug +#keys=console,luigi_debug + +[handler_console] +level=INFO +# class=StreamHandler +class=eventlog.TextHandler +args=(sys.stderr, ['execute', 'execute commit', 'complete query', 'find chunks']) +# args=(3, 0.1) +formatter=timed + +[handler_detail] +level=INFO +class=FileHandler +# use detail_log_dir rather than dir to facilitate stream editing +detail_log_dir=log +detail_log_file=%(detail_log_dir)s/cdm-detail.json +# append to log file +#args=('%(detail_log_file)s', 'a', None, True) +# overwrite log file +args=('%(detail_log_file)s', 'w') +formatter=json + +[handler_luigi_debug] +level=DEBUG +class=FileHandler +luigi_debug_log_dir=log +luigi_debug_log_file=%(luigi_debug_log_dir)s/cdm-luigi-debug.json +# append to log file +# args=('%(luigi_debug_log_file)s', 'a') +# overwrite log file +args=('%(luigi_debug_log_file)s', 'w') +formatter=json + +[formatters] +keys=timed, json + +[formatter_timed] +class=logging.Formatter +# %(name)s? +format=%(asctime)s %(process)s %(levelname)s: %(message)s +#datefmt=%02H:%02M:%02S + +[formatter_json] +class = pythonjsonlogger.jsonlogger.JsonFormatter +format=%(asctime)s %(process)s %(name) %(levelname): %(message)s %(args)s +#datefmt=%Y-%m-%02d %02H:%02M:%02S diff --git a/param_val.py b/param_val.py new file mode 100644 index 0000000..1b4c4aa --- /dev/null +++ b/param_val.py @@ -0,0 +1,37 @@ +# TODO: module doc: luigi's abuse of class attributes vs. mypy static typing +from datetime import datetime +from typing import Any, Callable, TypeVar, cast + +import luigi + + +_T = TypeVar('_T') +_U = TypeVar('_U') + + +def _valueOf(example: _T, cls: Callable[..., _U]) -> Callable[..., _T]: + + def getValue(*args: Any, **kwargs: Any) -> _T: + return cast(_T, cls(*args, **kwargs)) + return getValue + + +class TimeStampParameter(luigi.Parameter): + '''A datetime interchanged as milliseconds since the epoch. + ''' + + def parse(self, s: str) -> datetime: + ms = int(s) + return datetime.fromtimestamp(ms / 1000.0) + + def serialize(self, dt: datetime) -> str: + epoch = datetime.utcfromtimestamp(0) + ms = (dt - epoch).total_seconds() * 1000 + return str(int(ms)) + + +StrParam = _valueOf('s', luigi.Parameter) +IntParam = _valueOf(0, luigi.IntParameter) +BoolParam = _valueOf(True, luigi.BoolParameter) +DictParam = _valueOf({'k': 'v'}, luigi.DictParameter) +TimeStampParam = _valueOf(datetime(2001, 1, 1, 0, 0, 0), TimeStampParameter) diff --git a/pythonjsonlogger/__init__.py b/pythonjsonlogger/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pythonjsonlogger/jsonlogger.py b/pythonjsonlogger/jsonlogger.py new file mode 100644 index 0000000..4f96331 --- /dev/null +++ b/pythonjsonlogger/jsonlogger.py @@ -0,0 +1,141 @@ +''' +This library is provided to allow standard python logging +to output log data as JSON formatted strings +''' +import logging +import json +import re +import datetime +import traceback + +from inspect import istraceback + +#Support order in python 2.7 and 3 +try: + from collections import OrderedDict +except ImportError: + pass + +# skip natural LogRecord attributes +# http://docs.python.org/library/logging.html#logrecord-attributes +RESERVED_ATTRS = ( + 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename', + 'funcName', 'levelname', 'levelno', 'lineno', 'module', + 'msecs', 'message', 'msg', 'name', 'pathname', 'process', + 'processName', 'relativeCreated', 'stack_info', 'thread', 'threadName') + +RESERVED_ATTR_HASH = dict(zip(RESERVED_ATTRS, RESERVED_ATTRS)) + + +def merge_record_extra(record, target, reserved=RESERVED_ATTR_HASH): + """ + Merges extra attributes from LogRecord object into target dictionary + + :param record: logging.LogRecord + :param target: dict to update + :param reserved: dict or list with reserved keys to skip + """ + for key, value in record.__dict__.items(): + #this allows to have numeric keys + if (key not in reserved + and not (hasattr(key, "startswith") + and key.startswith('_'))): + target[key] = value + return target + + +class JsonFormatter(logging.Formatter): + """ + A custom formatter to format logging records as json strings. + extra values will be formatted as str() if nor supported by + json default encoder + """ + + def __init__(self, *args, **kwargs): + """ + :param json_default: a function for encoding non-standard objects + as outlined in http://docs.python.org/2/library/json.html + :param json_encoder: optional custom encoder + :param json_serializer: a :meth:`json.dumps`-compatible callable + that will be used to serialize the log record. + :param prefix: an optional string prefix added at the beginning of + the formatted string + """ + self.json_default = kwargs.pop("json_default", None) + self.json_encoder = kwargs.pop("json_encoder", None) + self.json_serializer = kwargs.pop("json_serializer", json.dumps) + self.prefix = kwargs.pop("prefix", "") + #super(JsonFormatter, self).__init__(*args, **kwargs) + logging.Formatter.__init__(self, *args, **kwargs) + if not self.json_encoder and not self.json_default: + def _default_json_handler(obj): + '''Prints dates in ISO format''' + if isinstance(obj, (datetime.date, datetime.time)): + return obj.isoformat() + elif istraceback(obj): + tb = ''.join(traceback.format_tb(obj)) + return tb.strip() + elif isinstance(obj, Exception): + return "Exception: %s" % str(obj) + return str(obj) + self.json_default = _default_json_handler + self._required_fields = self.parse() + self._skip_fields = dict(zip(self._required_fields, + self._required_fields)) + self._skip_fields.update(RESERVED_ATTR_HASH) + + def parse(self): + """Parses format string looking for substitutions""" + standard_formatters = re.compile(r'\((.+?)\)', re.IGNORECASE) + return standard_formatters.findall(self._fmt) + + def add_fields(self, log_record, record, message_dict): + """ + Override this method to implement custom logic for adding fields. + """ + for field in self._required_fields: + log_record[field] = record.__dict__.get(field) + log_record.update(message_dict) + merge_record_extra(record, log_record, reserved=self._skip_fields) + + def process_log_record(self, log_record): + """ + Override this method to implement custom logic + on the possibly ordered dictionary. + """ + return log_record + + def jsonify_log_record(self, log_record): + """Returns a json string of the log record.""" + return self.json_serializer(log_record, + default=self.json_default, + cls=self.json_encoder) + + def format(self, record): + """Formats a log record and serializes to json""" + message_dict = {} + if isinstance(record.msg, dict): + message_dict = record.msg + record.message = None + else: + record.message = record.getMessage() + # only format time if needed + if "asctime" in self._required_fields: + record.asctime = self.formatTime(record, self.datefmt) + + # Display formatted exception, but allow overriding it in the + # user-supplied dict. + if record.exc_info and not message_dict.get('exc_info'): + message_dict['exc_info'] = self.formatException(record.exc_info) + if not message_dict.get('exc_info') and record.exc_text: + message_dict['exc_info'] = record.exc_text + + try: + log_record = OrderedDict() + except NameError: + log_record = {} + + self.add_fields(log_record, record, message_dict) + log_record = self.process_log_record(log_record) + + return "%s%s" % (self.prefix, self.jsonify_log_record(log_record)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2a5e594 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,56 @@ +# This is a python packaging requirements file. +# https://packaging.python.org/tutorials/installing-packages/#requirements-files + +# Usage (development): +# +# $ virtualenv --python python3 ~/pyenv/luigi_cdm +# $ . ~/pyenv/luigi_cdm/bin/activate +# (luigi_cdm)$ pip install -r requirements.txt +# ... +# Successfully built luigi tornado +# Installing collected packages: ... +# ... tornado-4.5.3 + + +# Production usage is more like: +# +# $ docker run --rm -ti --entrypoint=/bin/bash stockport/luigi-taskrunner +# app@8ee96e02e548:/$ . /luigi/.pyenv/bin/activate +# (.pyenv) app@8ee96e02e548:/$ python --version +# Python 3.5.2 +# (.pyenv) app@8ee96e02e548:/$ pip freeze +# alembic==0.8.9 + +# cx_Oracle isn't pure-python nor open source, so installation is +# failure-prone. Fortunately, the stockport/luigi-taskrunner Docker +# image provides it: + +#Changes from 5 to 6.0.3 have created a twophase incompatibility between +#SQLAlchemy and cx-Oracle. +cx_Oracle==6.0.3 + +# Cython==0.25.1 +# docutils==0.12 +# lockfile==0.12.2 +luigi==2.7.1 +# Mako==1.0.6 +# MarkupSafe==0.23 +# numpy==1.11.2 +pandas==0.19.1 +# psycopg2==2.6.2 +# pymssql==2.1.3 +# PyPDF2==1.26.0 +# python-daemon==2.1.2 +# python-dateutil==2.6.0 +# python-editor==1.0.3 +# pytz==2016.10 +# requests==2.12.3 +# six==1.10.0 +# SQLAlchemy==1.1.4 has a twophase conflict with cx_Oracle==6.0.3 +SQLAlchemy==1.2.1 +#tornado==4.4.2 +#XlsxWriter==0.9.4 + +# let's "vendor" it instead, since +# we have no Internet access from our build jobs. +# python-json-logger==0.1.7 diff --git a/script_lib.py b/script_lib.py new file mode 100644 index 0000000..cfdff30 --- /dev/null +++ b/script_lib.py @@ -0,0 +1,320 @@ +r'''script_lib -- library of SQL scripts + +Scripts are pkg_resources, i.e. design-time constants. + +Each script should have a title, taken from the first line:: + + >>> Script.med_admin.title + 'create and populate the med_admin table.' + + >>> text = Script.med_admin.value + >>> lines = text.split('\n') + >>> print(lines[0]) + ... #doctest: +NORMALIZE_WHITESPACE + /** med_admin - create and populate the med_admin table. + +We can separate the script into statements:: + + >>> statements = Script.med_admin.statements() + >>> print(next(s for s in statements if 'insert' in s)) + ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + /** ... */ + insert into cdm_status (task, start_time) select 'med_admin', sysdate from dual + +A bit of sqlplus syntax is supported for ignoring errors in just part +of a script: + + >>> Script.sqlerror('whenever sqlerror exit') + False + >>> Script.sqlerror('whenever sqlerror continue') + True + >>> Script.sqlerror('select 1 + 1 from dual') is None + True + +The last statement should be a scalar query that returns non-zero to +signal that the script is complete: + + >>> print(statements[-1]) + ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + select records from cdm_status where task = 'med_admin' + +The completion test may depend on a digest of the script and its dependencies: + + >>> Script.med_admin.digest() != 0 + True + +ISSUE : Python hashes are senstive to the machine running the test? + +Some scripts use variables that are not known until a task is run; for +example, `&&upload_id` is used in names of objects such as tables and +partitions; these scripts must not refer to such variables in their +completion query: + + ..> del variables['upload_id'] + ..> print(Script.migrate_fact_upload.statements(variables, + ... skip_unbound=True)[-1].strip()) + commit + +''' + +from itertools import groupby +from typing import Dict, Iterable, List, Optional, Sequence, Text, Tuple, Type +from zlib import adler32 +import enum +import re +import abc + +import pkg_resources as pkg + +import sql_syntax +from sql_syntax import ( + Environment, StatementInContext, ObjectId, SQL, Name, + iter_statement) + +I2B2STAR = 'I2B2STAR' # cf. &&I2B2STAR in sql_scripts +CMS_RIF = 'CMS_RIF' + +ScriptStep = Tuple[int, Text, SQL] +Filename = str + + +class SQLMixin(enum.Enum): + @property + def sql(self) -> SQL: + from typing import cast + return cast(SQL, self.value) # hmm... + + @property + def fname(self) -> str: + return self.name + self.extension + + @abc.abstractproperty + def extension(self) -> str: + raise NotImplementedError + + @abc.abstractmethod + def parse(self, text: SQL) -> Iterable[StatementInContext]: + raise NotImplementedError + + def each_statement(self, + variables: Optional[Environment]=None, + skip_unbound: bool=False) -> Iterable[ScriptStep]: + for line, comment, statement in self.parse(self.sql): + try: + ss = sql_syntax.substitute(statement, self._all_vars(variables)) + except KeyError: + if skip_unbound: + continue + else: + raise + yield line, comment, ss + + def _all_vars(self, variables: Optional[Environment]) -> Optional[Environment]: + '''Add design_digest to variables. + ''' + if variables is None: + return None + return dict(variables, design_digest=str(self.digest())) + + def statements(self, + variables: Optional[Environment]=None, + skip_unbound: bool=False) -> Sequence[Text]: + return list(stmt for _l, _c, stmt + in self.each_statement(skip_unbound=skip_unbound, + variables=variables)) + + def created_objects(self) -> List[ObjectId]: + return [] + + def inserted_tables(self, + variables: Environment={}) -> List[Name]: + return [] + + @property + def title(self) -> Text: + line1 = self.sql.split('\n', 1)[0] + if not (line1.startswith('/** ') and ' - ' in line1): + raise ValueError('%s missing title block' % self) + return line1.split(' - ', 1)[1].strip() + + def deps(self) -> List['SQLMixin']: + return [child + for sql in self.statements() + for child in Script._get_deps(sql)] + + def dep_closure(self) -> List['SQLMixin']: + return [self] + [descendant + for sql in self.statements() + for child in Script._get_deps(sql) + for descendant in child.dep_closure()] + + def digest(self) -> int: + '''Hash the text of this script and its dependencies. + + Unlike the python hash() function, this digest is consistent across runs. + ''' + return adler32(str(self._text()).encode('utf-8')) + + def _text(self) -> List[str]: + '''Get the text of this script and its dependencies. + + >>> nodeps = Script.med_admin + >>> nodeps._text() == [nodeps.value] + True + + ..> complex = Script.epic_flowsheets_transform + ..> complex._text() != [complex.value] + True + ''' + return sorted(set(s.sql for s in self.dep_closure())) + + @classmethod + def _get_deps(cls, sql: Text) -> List['SQLMixin']: + ''' + >>> ds = Script._get_deps( + ... "select col from t where 'dep' = 'oops.sql'") + Traceback (most recent call last): + ... + KeyError: 'oops' + + >>> Script._get_deps( + ... "select col from t where x = 'name.sql'") + [] + ''' + from typing import cast + + m = re.search(r"select \S+ from \S+ where 'dep' = '([^']+)'", sql) + if not m: + return [] + name, ext = m.group(1).rsplit('.', 1) + choices = list(Script) if ext == 'sql' else [] + deps = [cast(SQLMixin, s) for s in choices if s.name == name] + if not deps: + raise KeyError(name) + return deps + + @classmethod + def sqlerror(cls, s: SQL) -> Optional[bool]: + if s.strip().lower() == 'whenever sqlerror exit': + return False + elif s.strip().lower() == 'whenever sqlerror continue': + return True + return None + + +class ScriptMixin(SQLMixin): + @property + def extension(self) -> str: + return '.sql' + + def parse(self, text: SQL, + block_sep: str='/') -> Iterable[StatementInContext]: + lines = [l.strip() for l in text.split('\n')] + return (sql_syntax.iter_blocks(text) if block_sep in lines + else iter_statement(text)) + + def created_objects(self) -> List[ObjectId]: + return [obj + for _l, _comment, stmt in iter_statement(self.sql) + for obj in sql_syntax.created_objects(stmt)] + + def inserted_tables(self, + variables: Optional[Environment]={}) -> List[Name]: + return [obj + for _l, _comment, stmt in iter_statement(self.sql) + for obj in sql_syntax.inserted_tables( + sql_syntax.substitute(stmt, self._all_vars(variables)))] + + +class Script(ScriptMixin, enum.Enum): + '''Script is an enum.Enum of contents. + + ISSUE: It's tempting to consider separate libraries for NAACCR, + NTDS, etc., but that doesn't integrate well with the + generic luigi.EnumParameter() in etl_tasks.SqlScriptTask. + + ''' + [ + # Keep sorted + condition, + death, + death_cause, + demographic, + diagnosis, + dispensing, + encounter, + enrollment, + harvest, + lab_result_cm, + med_admin, + med_admin_init, + obs_clin, + obs_gen, + patient_chunks_survey, + pcornet_init, + pcornet_loader, + pcornet_trial, + prescribing, + pro_cm, + procedures, + provider, + vital + ] = [ + pkg.resource_string(__name__, + 'Oracle/' + fname).decode('utf-8') + for fname in [ + 'condition.sql', + 'death.sql', + 'death_cause.sql', + 'demographic.sql', + 'diagnosis.sql', + 'dispensing.sql', + 'encounter.sql', + 'enrollment.sql', + 'harvest.sql', + 'lab_result_cm.sql', + 'med_admin.sql', + 'med_admin_init.sql', + 'obs_clin.sql', + 'obs_gen.sql', + 'patient_chunks_survey.sql', + 'pcornet_init.sql', + 'pcornet_loader.sql', + 'pcornet_trial.sql', + 'prescribing.sql', + 'pro_cm.sql', + 'procedures.sql', + 'provider.sql', + 'vital.sql' + ] + ] + + def __repr__(self) -> str: + return '<%s(%s)>' % (self.__class__.__name__, self.name) + + +def _object_to_creators(libs: List[Type[SQLMixin]]) -> Dict[ObjectId, List[SQLMixin]]: + '''Find creator scripts for each object. + + "There can be only one." + >>> creators = _object_to_creators([Script]) + >>> [obj for obj, scripts in creators.items() + ... if len(scripts) > 1] + [] + ''' + fst = lambda pair: pair[0] + snd = lambda pair: pair[1] + + objs = sorted( + [(obj, s) + for lib in libs for s in lib + for obj in s.created_objects()], + key=fst) + by_obj = groupby(objs, key=fst) + return dict((obj, list(map(snd, places))) for obj, places in by_obj) + + +_redefined_objects = [ + obj for obj, scripts in _object_to_creators([Script]).items() + if len(scripts) > 1] +assert _redefined_objects == [], _redefined_objects diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..493067d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,53 @@ +# Usage: +# +# $ nosetests && flake8 . && mypy . + + +[nosetests] +# verbosity=3 +with-doctest=1 +where=. + +[flake8] +# E731: Assigning to lambda seems OK +# E126: Emacs python mode seems to over-indent? +# W605: TODO: good idea, but our code was written before the tool pointed this out. +# E252: TODO: community style seems to have changed after we wrote our code. +# W504: er.. you complain about line breaks on both sides. what are we supposed to do? +# E741: ambiguous variable name. TODO: good idea, but wasn't there when we wrote the code +ignore = E731, E126, W605, E252, W504, E741 + +# with type annotations, 79 is awkward +# guide, for window sizing: +# 23456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 +max-line-length = 120 +exclude = pythonjsonlogger,ADD_SCILHS_100,.direnv + + +[mypy] +# http://mypy.readthedocs.io/en/latest/config_file.html +mypy_path=stubs +warn_redundant_casts=true +warn_unused_ignores=true +strict_optional=true + +# but all the other --strict flags (from mypy -h): +disallow_untyped_calls=true +disallow_untyped_defs=true +check_untyped_defs=true +warn_return_any=true + +[mypy-luigi.*,cx_Oracle.*,sqlalchemy.*] +disallow_untyped_defs=false + +[mypy-spreadsync.*] +ignore_errors = True + +[mypy-sqlalchemy.*] +ignore_errors = True + +[mypy-pythonjsonlogger.*] +ignore_errors = True + +#--cobertura-xml-report DIR +#--junit-xml JUNIT_XML diff --git a/sql-style.xml b/sql-style.xml new file mode 100644 index 0000000..6e1e56a --- /dev/null +++ b/sql-style.xml @@ -0,0 +1,453 @@ + + + + + 1:kumc-bmi-sql-style + + false + false + false + true + true + true + false + false + 1 + true + false + false + false + true + true + false + true + true + true + true + false + true + true + false + 2 + false + false + 3 + false + false + true + 10 + 120 + + false + kumc-bmi-sql-style + 1 + 2 + 2 + 2 + 0 + false + 60 + 1 + 0 + 0 + false + false + 0 + 3 + false + + + + 1:Old Preferences + + true + false + true + true + true + true + false + true + 0 + false + false + true + false + true + false + false + true + true + true + true + true + true + true + false + 0 + false + false + 0 + false + true + false + 999 + 999 + + false + Old Preferences + 1 + 2 + 0 + 0 + 0 + false + 80 + 1 + 0 + 0 + false + false + 0 + 0 + false + + + + 1:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 1 + 0 + 0 + false + false + 0 + 0 + false + + + + 2:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 2 + 0 + 0 + false + false + 0 + 0 + false + + + + 3:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 3 + 0 + 0 + false + false + 0 + 0 + false + + + + 4:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 4 + 0 + 0 + false + false + 0 + 0 + false + + + + 5:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 5 + 0 + 0 + false + false + 0 + 0 + false + + + + 6:SQL + + true + false + true + true + true + true + false + true + 0 + true + false + true + false + true + false + false + true + true + true + true + true + true + true + true + 0 + true + false + 0 + false + true + false + 10 + 80 + + false + SQL + 1 + 2 + 0 + 0 + 0 + false + 20 + 6 + 0 + 0 + false + false + 0 + 0 + false + + + + diff --git a/sql_syntax.py b/sql_syntax.py new file mode 100644 index 0000000..c5e5a6b --- /dev/null +++ b/sql_syntax.py @@ -0,0 +1,238 @@ +'''sql_syntax - break SQL scripts into statements, etc. + +''' + +from datetime import datetime +from typing import Dict, Iterable, List, Optional, Text, Tuple, Union +import re + +Name = Text +SQL = Text +Environment = Dict[Name, Text] +Params = Dict[str, Union[str, int, datetime]] +Line = int +Comment = Text +StatementInContext = Tuple[Line, Comment, SQL] + + +def iter_statement(txt: SQL) -> Iterable[StatementInContext]: + r'''Iterate over SQL statements in a script. + + >>> list(iter_statement("drop table foo; create table foo")) + [(1, '', 'drop table foo'), (1, '', 'create table foo')] + + >>> list(iter_statement("-- blah blah\ndrop table foo")) + [(2, '-- blah blah\n', 'drop table foo')] + + >>> list(iter_statement("drop /* blah blah */ table foo")) + [(1, '', 'drop table foo')] + + >>> list(iter_statement("select '[^;]+' from dual")) + [(1, '', "select '[^;]+' from dual")] + + >>> list(iter_statement('select "x--y" from z;')) + [(1, '', 'select "x--y" from z')] + >>> list(iter_statement("select 'x--y' from z;")) + [(1, '', "select 'x--y' from z")] + ''' + + statement = comment = '' + line = 1 + sline = None # type: Optional[int] + + def save(txt: SQL) -> Tuple[SQL, Optional[int]]: + return (statement + txt, sline or (line if txt else None)) + + while 1: + m = SQL_SEPARATORS.search(txt) + if not m: + statement, sline = save(txt) + break + + pfx, match, txt = (txt[:m.start()], + txt[m.start():m.end()], + txt[m.end():]) + if pfx: + statement, sline = save(pfx) + + if m.group('sep'): + if sline: + yield sline, comment, statement + statement = comment = '' + sline = None + elif [n for n in ('lit', 'hint', 'sym') + if m.group(n)]: + statement, sline = save(match) + elif (m.group('space') and statement): + statement, sline = save(match) + elif ((m.group('comment') and not statement) or + (m.group('space') and comment)): + comment += match + + line += (pfx + match).count("\n") + + if sline and (comment or statement): + yield sline, comment, statement + + +# Check for hint before comment since a hint looks like a comment +SQL_SEPARATORS = re.compile( + r'(?P^\s+)' + r'|(?P/\*\+.*?\*/)' + r'|(?P(--[^\n]*(?:\n|$))|(?:/\*([^\*]|(\*(?!/)))*\*/))' + r'|(?P"[^\"]*")' + r"|(?P'[^\']*')" + r'|(?P;)') + + +def _test_iter_statement() -> None: + r''' + >>> list(iter_statement("/* blah blah */ drop table foo")) + [(1, '/* blah blah */ ', 'drop table foo')] + + >>> [l for (l, c, s) in iter_statement('s1;\n/**********************\n' + ... '* Medication concepts \n' + ... '**********************/\ns2')] + [1, 5] + + >>> list(iter_statement("")) + [] + + >>> list(iter_statement("/*...*/ ")) + [] + + >>> list(iter_statement("drop table foo; ")) + [(1, '', 'drop table foo')] + + >>> list(iter_statement("drop table foo")) + [(1, '', 'drop table foo')] + + >>> list(iter_statement("/* *** -- *** */ drop table foo")) + [(1, '/* *** -- *** */ ', 'drop table foo')] + + >>> list(iter_statement("/* *** -- *** */ drop table x /* ... */")) + [(1, '/* *** -- *** */ ', 'drop table x ')] + + >>> list(iter_statement('select /*+ index(ix1 ix2) */ * from some_table')) + [(1, '', 'select /*+ index(ix1 ix2) */ * from some_table')] + + >>> len(list(iter_statement(""" /* + no space before + in hints*/ + ... /*+ index(observation_fact fact_cnpt_pat_enct_idx) */ + ... select * from some_table; """))[0][2]) + 79 + + >>> list(iter_statement('select 1+1; /* nothing else */; ')) + [(1, '', 'select 1+1')] + ''' + pass # pragma: nocover + + +def substitute(sql: SQL, variables: Optional[Environment]) -> SQL: + '''Evaluate substitution variables in the style of Oracle sqlplus. + + >>> substitute('select &¬_bound from dual', {}) + Traceback (most recent call last): + KeyError: 'not_bound' + ''' + if variables is None: + return sql + sql_esc = sql.replace('%', '%%') # escape %, which we use specially + return re.sub('&&(\w+)', r'%(\1)s', sql_esc) % variables + + +def params_used(params: Params, statement: SQL) -> Params: + return dict((k, v) for (k, v) in params.items() + if k in param_names(statement)) + + +def param_names(s: SQL) -> List[Name]: + ''' + >>> param_names('select 1+1 from dual') + [] + >>> param_names('select 1+:y from dual') + ['y'] + ''' + return [expr[1:] + for expr in re.findall(r':\w+', s)] + + +def first_cursor(statement: SQL) -> SQL: + '''Find argument of first obvious call to `cursor()`. + + >>> first_cursor('select * from table(f(cursor(select * from there)))') + 'select * from there' + ''' + return statement.split('cursor(')[1].split(')')[0] + + +class ObjectId(object): + kind = '' + + def __init__(self, name: str) -> None: + self.name = name + + def __repr__(self) -> str: + return '{} {}'.format(self.kind, self.name) + + def __hash__(self) -> int: + return hash((self.kind, self.name)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ObjectId): + return False + return (self.kind, self.name) == ((other.kind, other.name)) + + def __lt__(self, other: 'ObjectId') -> bool: + return (self.kind, self.name) < ((other.kind, other.name)) + + +class TableId(ObjectId): + kind = 'table' + + +class ViewId(ObjectId): + kind = 'view' + + +def created_objects(statement: SQL) -> List[ObjectId]: + r''' + >>> created_objects('create table t as ...') + [table t] + + >>> created_objects('create or replace view x\nas ...') + [view x] + ''' + m = re.search('(?i)^create or replace view (\S+)', statement.strip()) + views = [ViewId(m.group(1))] if m else [] # type: List[ObjectId] + m = re.search('(?i)^create table (\w+)', statement.strip()) + tables = [TableId(m.group(1))] if m else [] # type: List[ObjectId] + return tables + views + + +def inserted_tables(statement: SQL) -> List[Name]: + r''' + >>> inserted_tables('create table t as ...') + [] + >>> inserted_tables('insert into t (...) ...') + ['t'] + ''' + if not statement.startswith('insert'): + return [] + m = re.search('into\s+(\S+)', statement.strip()) + return [m.group(1)] if m else [] + + +def insert_append_table(statement: SQL) -> Optional[Name]: + if '/*+ append' in statement: + [t] = inserted_tables(statement) + return t + return None + + +def iter_blocks(module: SQL, + separator: str =r'\r?\n/\r?\n') -> Iterable[StatementInContext]: + line = 1 + for block in re.split(separator, module): + if block.strip(): + yield (line, '...no comment handling...', block) + line += len((block + separator).split('\n')[:-1]) diff --git a/stubs/cx_Oracle.pyi b/stubs/cx_Oracle.pyi new file mode 100644 index 0000000..f96e852 --- /dev/null +++ b/stubs/cx_Oracle.pyi @@ -0,0 +1,426 @@ +# Stubs for cx_Oracle (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from datetime import date +from typing import Any + +ATTR_PURITY_DEFAULT = ... # type: int +ATTR_PURITY_NEW = ... # type: int +ATTR_PURITY_SELF = ... # type: int +DBSHUTDOWN_ABORT = ... # type: int +DBSHUTDOWN_FINAL = ... # type: int +DBSHUTDOWN_IMMEDIATE = ... # type: int +DBSHUTDOWN_TRANSACTIONAL = ... # type: int +DBSHUTDOWN_TRANSACTIONAL_LOCAL = ... # type: int +EVENT_DEREG = ... # type: int +EVENT_NONE = ... # type: int +EVENT_OBJCHANGE = ... # type: int +EVENT_QUERYCHANGE = ... # type: int +EVENT_SHUTDOWN = ... # type: int +EVENT_SHUTDOWN_ANY = ... # type: int +EVENT_STARTUP = ... # type: int +FNCODE_BINDBYNAME = ... # type: int +FNCODE_BINDBYPOS = ... # type: int +FNCODE_DEFINEBYPOS = ... # type: int +FNCODE_STMTEXECUTE = ... # type: int +FNCODE_STMTFETCH = ... # type: int +FNCODE_STMTPREPARE = ... # type: int +OPCODE_ALLOPS = ... # type: int +OPCODE_ALLROWS = ... # type: int +OPCODE_ALTER = ... # type: int +OPCODE_DELETE = ... # type: int +OPCODE_DROP = ... # type: int +OPCODE_INSERT = ... # type: int +OPCODE_UPDATE = ... # type: int +PRELIM_AUTH = ... # type: int +SPOOL_ATTRVAL_FORCEGET = ... # type: int +SPOOL_ATTRVAL_NOWAIT = ... # type: int +SPOOL_ATTRVAL_WAIT = ... # type: int +SUBSCR_CQ_QOS_BEST_EFFORT = ... # type: int +SUBSCR_CQ_QOS_CLQRYCACHE = ... # type: int +SUBSCR_CQ_QOS_QUERY = ... # type: int +SUBSCR_NAMESPACE_DBCHANGE = ... # type: int +SUBSCR_PROTO_HTTP = ... # type: int +SUBSCR_PROTO_MAIL = ... # type: int +SUBSCR_PROTO_OCI = ... # type: int +SUBSCR_PROTO_SERVER = ... # type: int +SUBSCR_QOS_HAREG = ... # type: int +SUBSCR_QOS_MULTICBK = ... # type: int +SUBSCR_QOS_PAYLOAD = ... # type: int +SUBSCR_QOS_PURGE_ON_NTFN = ... # type: int +SUBSCR_QOS_RELIABLE = ... # type: int +SUBSCR_QOS_REPLICATE = ... # type: int +SUBSCR_QOS_SECURE = ... # type: int +SYSASM = ... # type: int +SYSDBA = ... # type: int +SYSOPER = ... # type: int +UCBTYPE_ENTRY = ... # type: int +UCBTYPE_EXIT = ... # type: int +UCBTYPE_REPLACE = ... # type: int +apilevel = ... # type: str +buildtime = ... # type: str +paramstyle = ... # type: str +threadsafety = ... # type: int +version = ... # type: str + +def DateFromTicks(*args, **kwargs): ... +def Time(*args, **kwargs): ... +def TimeFromTicks(*args, **kwargs): ... +def TimestampFromTicks(*args, **kwargs): ... +def clientversion(*args, **kwargs): ... +def makedsn(*args, **kwargs): ... + +class _BASEVARTYPE(object): ... + +class BFILE(_BASEVARTYPE): ... + +class BINARY(_BASEVARTYPE): ... + +class BLOB(_BASEVARTYPE): ... + +class Binary: + maketrans = ... # type: Any + def __init__(self, *args, **kwargs): ... + def capitalize(self, *args, **kwargs): ... + def center(self, *args, **kwargs): ... + def count(self, *args, **kwargs): ... + def decode(self, *args, **kwargs): ... + def endswith(self, *args, **kwargs): ... + def expandtabs(self, *args, **kwargs): ... + def find(self, *args, **kwargs): ... + @classmethod + def fromhex(cls, *args, **kwargs): ... + def hex(self, *args, **kwargs): ... + def index(self, *args, **kwargs): ... + def isalnum(self, *args, **kwargs): ... + def isalpha(self, *args, **kwargs): ... + def isdigit(self, *args, **kwargs): ... + def islower(self, *args, **kwargs): ... + def isspace(self, *args, **kwargs): ... + def istitle(self, *args, **kwargs): ... + def isupper(self, *args, **kwargs): ... + def join(self, *args, **kwargs): ... + def ljust(self, *args, **kwargs): ... + def lower(self, *args, **kwargs): ... + def lstrip(self, *args, **kwargs): ... + def partition(self, *args, **kwargs): ... + def replace(self, *args, **kwargs): ... + def rfind(self, *args, **kwargs): ... + def rindex(self, *args, **kwargs): ... + def rjust(self, *args, **kwargs): ... + def rpartition(self, *args, **kwargs): ... + def rsplit(self, *args, **kwargs): ... + def rstrip(self, *args, **kwargs): ... + def split(self, *args, **kwargs): ... + def splitlines(self, *args, **kwargs): ... + def startswith(self, *args, **kwargs): ... + def strip(self, *args, **kwargs): ... + def swapcase(self, *args, **kwargs): ... + def title(self, *args, **kwargs): ... + def translate(self, *args, **kwargs): ... + def upper(self, *args, **kwargs): ... + def zfill(self, *args, **kwargs): ... + def __add__(self, other): ... + def __contains__(self, *args, **kwargs): ... + def __eq__(self, other): ... + def __ge__(self, other): ... + def __getitem__(self, index): ... + def __getnewargs__(self, *args, **kwargs): ... + def __gt__(self, other): ... + def __hash__(self): ... + def __iter__(self): ... + def __le__(self, other): ... + def __len__(self, *args, **kwargs): ... + def __lt__(self, other): ... + def __mod__(self, other): ... + def __mul__(self, other): ... + def __ne__(self, other): ... + def __rmod__(self, other): ... + def __rmul__(self, other): ... + +class CLOB(_BASEVARTYPE): ... + +class CURSOR(_BASEVARTYPE): ... + +class Connection: + action = ... # type: Any + autocommit = ... # type: Any + client_identifier = ... # type: Any + clientinfo = ... # type: Any + current_schema = ... # type: Any + dsn = ... # type: Any + encoding = ... # type: Any + inputtypehandler = ... # type: Any + maxBytesPerCharacter = ... # type: Any + module = ... # type: Any + nencoding = ... # type: Any + outputtypehandler = ... # type: Any + stmtcachesize = ... # type: Any + tnsentry = ... # type: Any + username = ... # type: Any + version = ... # type: Any + def __init__(self, *args, **kwargs): ... + def begin(self, *args, **kwargs): ... + def cancel(self, *args, **kwargs): ... + def changepassword(self, *args, **kwargs): ... + def close(self, *args, **kwargs): ... + def commit(self, *args, **kwargs): ... + def cursor(self, *args, **kwargs): ... + def ping(self, *args, **kwargs): ... + def prepare(self, *args, **kwargs): ... + def register(self, *args, **kwargs): ... + def rollback(self, *args, **kwargs): ... + def shutdown(self, *args, **kwargs): ... + def startup(self, *args, **kwargs): ... + def subscribe(self, *args, **kwargs): ... + def unregister(self, *args, **kwargs): ... + def __enter__(self, *args, **kwargs): ... + def __exit__(self, *args, **kwargs): ... + +class Cursor: + arraysize = ... # type: Any + bindarraysize = ... # type: Any + bindvars = ... # type: Any + connection = ... # type: Any + description = ... # type: Any + fetchvars = ... # type: Any + inputtypehandler = ... # type: Any + numbersAsStrings = ... # type: Any + outputtypehandler = ... # type: Any + rowcount = ... # type: Any + rowfactory = ... # type: Any + statement = ... # type: Any + def __init__(self, *args, **kwargs): ... + def arrayvar(self, *args, **kwargs): ... + def bindnames(self, *args, **kwargs): ... + def callfunc(self, *args, **kwargs): ... + def callproc(self, *args, **kwargs): ... + def close(self, *args, **kwargs): ... + def execute(self, *args, **kwargs): ... + def executemany(self, *args, **kwargs): ... + def executemanyprepared(self, *args, **kwargs): ... + def fetchall(self, *args, **kwargs): ... + def fetchmany(self, *args, **kwargs): ... + def fetchone(self, *args, **kwargs): ... + def fetchraw(self, *args, **kwargs): ... + def getbatcherrors(self, *args, **kwargs): ... + def parse(self, *args, **kwargs): ... + def prepare(self, *args, **kwargs): ... + def setinputsizes(self, *args, **kwargs): ... + def setoutputsize(self, *args, **kwargs): ... + def var(self, *args, **kwargs): ... + def __iter__(self): ... + def __next__(self): ... + +class DATETIME(_BASEVARTYPE): ... + +class DataError(DatabaseError): ... + +class DatabaseError(Error): ... + +class Date: + day = ... # type: Any + max = ... # type: Any + min = ... # type: Any + month = ... # type: Any + resolution = ... # type: Any + year = ... # type: Any + def __init__(self, *args, **kwargs): ... + def ctime(self, *args, **kwargs): ... + @classmethod + def fromordinal(cls, *args, **kwargs): ... + @classmethod + def fromtimestamp(cls, *args, **kwargs): ... + def isocalendar(self, *args, **kwargs): ... + def isoformat(self, *args, **kwargs): ... + def isoweekday(self, *args, **kwargs): ... + def replace(self, *args, **kwargs): ... + def strftime(self, *args, **kwargs): ... + def timetuple(self, *args, **kwargs): ... + @classmethod + def today(cls, *args, **kwargs): ... + def toordinal(self, *args, **kwargs): ... + def weekday(self, *args, **kwargs): ... + def __add__(self, other): ... + def __eq__(self, other): ... + def __format__(self, *args, **kwargs): ... + def __ge__(self, other): ... + def __gt__(self, other): ... + def __hash__(self): ... + def __le__(self, other): ... + def __lt__(self, other): ... + def __ne__(self, other): ... + def __radd__(self, other): ... + def __reduce__(self): ... + def __rsub__(self, other): ... + def __sub__(self, other): ... + +class Error(Exception): ... + +class FIXED_CHAR(_BASEVARTYPE): ... + +class FIXED_NCHAR(_BASEVARTYPE): ... + +class FIXED_UNICODE(_BASEVARTYPE): ... + +class INTERVAL(_BASEVARTYPE): ... + +class IntegrityError(DatabaseError): ... + +class InterfaceError(Error): ... + +class InternalError(DatabaseError): ... + +class LOB: + def close(self, *args, **kwargs): ... + def fileexists(self, *args, **kwargs): ... + def getchunksize(self, *args, **kwargs): ... + def getfilename(self, *args, **kwargs): ... + def isopen(self, *args, **kwargs): ... + def open(self, *args, **kwargs): ... + def read(self, *args, **kwargs): ... + def setfilename(self, *args, **kwargs): ... + def size(self, *args, **kwargs): ... + def trim(self, *args, **kwargs): ... + def write(self, *args, **kwargs): ... + def __reduce__(self): ... + +class LONG_BINARY(_BASEVARTYPE): ... + +class LONG_NCHAR(_BASEVARTYPE): ... + +class LONG_STRING(_BASEVARTYPE): ... + +class LONG_UNICODE(_BASEVARTYPE): ... + +class NATIVE_FLOAT(_BASEVARTYPE): ... + +class NCHAR(_BASEVARTYPE): ... + +class NCLOB(_BASEVARTYPE): ... + +class NUMBER(_BASEVARTYPE): ... + +class NotSupportedError(DatabaseError): ... + +class OBJECT(_BASEVARTYPE): + type = ... # type: Any + +class OperationalError(DatabaseError): ... + +class ProgrammingError(DatabaseError): ... + +class ROWID(_BASEVARTYPE): ... + +class STRING(_BASEVARTYPE): ... + +class SessionPool: + busy = ... # type: Any + dsn = ... # type: Any + getmode = ... # type: Any + homogeneous = ... # type: Any + increment = ... # type: Any + max = ... # type: Any + min = ... # type: Any + name = ... # type: Any + opened = ... # type: Any + timeout = ... # type: Any + tnsentry = ... # type: Any + username = ... # type: Any + def __init__(self, *args, **kwargs): ... + def acquire(self, *args, **kwargs): ... + def drop(self, *args, **kwargs): ... + def release(self, *args, **kwargs): ... + +class TIMESTAMP(_BASEVARTYPE): ... + +class Timestamp(date): + hour = ... # type: Any + microsecond = ... # type: Any + minute = ... # type: Any + second = ... # type: Any + tzinfo = ... # type: Any + def __init__(self, *args, **kwargs): ... + def astimezone(self, *args, **kwargs): ... + @classmethod + def combine(cls, *args, **kwargs): ... + def ctime(self, *args, **kwargs): ... + def date(self, *args, **kwargs): ... + def dst(self, *args, **kwargs): ... + @classmethod + def fromtimestamp(cls, *args, **kwargs): ... + def isoformat(self, *args, **kwargs): ... + @classmethod + def now(cls, *args, **kwargs): ... + def replace(self, *args, **kwargs): ... + @classmethod + def strptime(cls, *args, **kwargs): ... + def time(self, *args, **kwargs): ... + def timestamp(self, *args, **kwargs): ... + def timetuple(self, *args, **kwargs): ... + def timetz(self, *args, **kwargs): ... + def tzname(self, *args, **kwargs): ... + @classmethod + def utcfromtimestamp(cls, *args, **kwargs): ... + @classmethod + def utcnow(cls, *args, **kwargs): ... + def utcoffset(self, *args, **kwargs): ... + def utctimetuple(self, *args, **kwargs): ... + def __add__(self, other): ... + def __eq__(self, other): ... + def __ge__(self, other): ... + def __gt__(self, other): ... + def __hash__(self): ... + def __le__(self, other): ... + def __lt__(self, other): ... + def __ne__(self, other): ... + def __radd__(self, other): ... + def __reduce__(self): ... + def __rsub__(self, other): ... + def __sub__(self, other): ... + +class UNICODE(_BASEVARTYPE): ... + +class Warning(Exception): ... + +class _Error: + code = ... # type: Any + context = ... # type: Any + message = ... # type: Any + offset = ... # type: Any + +class connect: + action = ... # type: Any + autocommit = ... # type: Any + client_identifier = ... # type: Any + clientinfo = ... # type: Any + current_schema = ... # type: Any + dsn = ... # type: Any + encoding = ... # type: Any + inputtypehandler = ... # type: Any + maxBytesPerCharacter = ... # type: Any + module = ... # type: Any + nencoding = ... # type: Any + outputtypehandler = ... # type: Any + stmtcachesize = ... # type: Any + tnsentry = ... # type: Any + username = ... # type: Any + version = ... # type: Any + def __init__(self, *args, **kwargs): ... + def begin(self, *args, **kwargs): ... + def cancel(self, *args, **kwargs): ... + def changepassword(self, *args, **kwargs): ... + def close(self, *args, **kwargs): ... + def commit(self, *args, **kwargs): ... + def cursor(self, *args, **kwargs): ... + def ping(self, *args, **kwargs): ... + def prepare(self, *args, **kwargs): ... + def register(self, *args, **kwargs): ... + def rollback(self, *args, **kwargs): ... + def shutdown(self, *args, **kwargs): ... + def startup(self, *args, **kwargs): ... + def subscribe(self, *args, **kwargs): ... + def unregister(self, *args, **kwargs): ... + def __enter__(self, *args, **kwargs): ... + def __exit__(self, *args, **kwargs): ... diff --git a/stubs/luigi/__init__.pyi b/stubs/luigi/__init__.pyi new file mode 100644 index 0000000..3a21420 --- /dev/null +++ b/stubs/luigi/__init__.pyi @@ -0,0 +1,20 @@ +# Stubs for luigi (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from luigi import task as task +from luigi.task import Task as Task, Config as Config, ExternalTask as ExternalTask, WrapperTask as WrapperTask, namespace as namespace, auto_namespace as auto_namespace +from luigi import target as target +from luigi.target import Target as Target +from luigi import local_target as local_target +from luigi.local_target import LocalTarget as LocalTarget +from luigi import rpc as rpc +from luigi.rpc import RemoteScheduler as RemoteScheduler, RPCError as RPCError +from luigi import parameter as parameter +from luigi.parameter import Parameter as Parameter, DateParameter as DateParameter, MonthParameter as MonthParameter, YearParameter as YearParameter, DateHourParameter as DateHourParameter, DateMinuteParameter as DateMinuteParameter, DateSecondParameter as DateSecondParameter, DateIntervalParameter as DateIntervalParameter, TimeDeltaParameter as TimeDeltaParameter, IntParameter as IntParameter, FloatParameter as FloatParameter, BoolParameter as BoolParameter, TaskParameter as TaskParameter, EnumParameter as EnumParameter, DictParameter as DictParameter, ListParameter as ListParameter, TupleParameter as TupleParameter, NumericalParameter as NumericalParameter, ChoiceParameter as ChoiceParameter +from luigi import configuration as configuration +from luigi import interface as interface +from luigi.interface import run as run, build as build +from luigi import event as event +from luigi.event import Event as Event +from .tools import range as range diff --git a/stubs/luigi/configuration.pyi b/stubs/luigi/configuration.pyi new file mode 100644 index 0000000..06a65a2 --- /dev/null +++ b/stubs/luigi/configuration.pyi @@ -0,0 +1,26 @@ +# Stubs for luigi.configuration (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional +# py2: from ConfigParser import ConfigParser +from configparser import ConfigParser + +class LuigiConfigParser(ConfigParser): + NO_DEFAULT = ... # type: Any + config_file = ... # type: Any + @classmethod + def add_config_path(cls, path): ... + @classmethod + def instance(cls, *args, **kwargs): ... + @classmethod + def reload(cls): ... + # avoid: Signature of "get" incompatible with supertype "Mapping" + def get(self, section, option, default: Any = ..., **kwargs): ... # type:ignore + def getboolean(self, section, option, default: Any = ...): ... # type:ignore + def getint(self, section, option, default: Any = ...): ... # type:ignore + def getfloat(self, section, option, default: Any = ...): ... # type:ignore + def getintdict(self, section): ... + def set(self, section, option, value: Optional[Any] = ...): ... + +def get_config(): ... diff --git a/stubs/luigi/contrib/__init__.pyi b/stubs/luigi/contrib/__init__.pyi new file mode 100644 index 0000000..1a7b73e --- /dev/null +++ b/stubs/luigi/contrib/__init__.pyi @@ -0,0 +1,4 @@ +# Stubs for luigi.contrib (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + diff --git a/stubs/luigi/contrib/sqla.pyi b/stubs/luigi/contrib/sqla.pyi new file mode 100644 index 0000000..eb90250 --- /dev/null +++ b/stubs/luigi/contrib/sqla.pyi @@ -0,0 +1,43 @@ +# Stubs for luigi.contrib.sqla (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Dict, Any +import luigi +from collections import namedtuple +from sqlalchemy.engine import Engine + +class SQLAlchemyTarget(luigi.Target): + marker_table = ... # type: Any + + Connection = namedtuple('Connection', 'engine pid') + target_table = ... # type: str + update_id = ... # type: str + connection_string = ... # type: str + echo = ... # type: bool + connect_args = ... # type: Dict[str, Any] + marker_table_bound = ... # type: Any + def __init__(self, connection_string, target_table, update_id, echo: bool = ..., connect_args: Any = ...) -> None: ... + @property + def engine(self) -> Engine: ... + def touch(self) -> None: ... + def exists(self) -> bool: ... + def create_marker_table(self) -> None: ... + def open(self, mode): ... + +class CopyToTable(luigi.Task): + echo = ... # type: bool + connect_args = ... # type: Any + def connection_string(self): ... + def table(self): ... + columns = ... # type: Any + column_separator = ... # type: str + chunk_size = ... # type: int + reflect = ... # type: bool + table_bound = ... # type: Any + def create_table(self, engine): ... + def update_id(self): ... + def output(self): ... + def rows(self): ... + def run(self): ... + def copy(self, conn, ins_rows, table_bound): ... diff --git a/stubs/luigi/event.pyi b/stubs/luigi/event.pyi new file mode 100644 index 0000000..1cfa689 --- /dev/null +++ b/stubs/luigi/event.pyi @@ -0,0 +1,15 @@ +# Stubs for luigi.event (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +class Event: + DEPENDENCY_DISCOVERED = ... # type: str + DEPENDENCY_MISSING = ... # type: str + DEPENDENCY_PRESENT = ... # type: str + BROKEN_TASK = ... # type: str + START = ... # type: str + FAILURE = ... # type: str + SUCCESS = ... # type: str + PROCESSING_TIME = ... # type: str + TIMEOUT = ... # type: str + PROCESS_FAILURE = ... # type: str diff --git a/stubs/luigi/interface.pyi b/stubs/luigi/interface.pyi new file mode 100644 index 0000000..950c658 --- /dev/null +++ b/stubs/luigi/interface.pyi @@ -0,0 +1,37 @@ +# Stubs for luigi.interface (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional +from . import task + +def setup_interface_logging(conf_file: str = ..., level_name: str = ...): ... + +class core(task.Config): + use_cmdline_section = ... # type: bool + local_scheduler = ... # type: Any + scheduler_host = ... # type: Any + scheduler_port = ... # type: Any + scheduler_url = ... # type: Any + lock_size = ... # type: Any + no_lock = ... # type: Any + lock_pid_dir = ... # type: Any + take_lock = ... # type: Any + workers = ... # type: Any + logging_conf_file = ... # type: Any + log_level = ... # type: Any + module = ... # type: Any + parallel_scheduling = ... # type: Any + assistant = ... # type: Any + help = ... # type: Any + help_all = ... # type: Any + +class _WorkerSchedulerFactory: + def create_local_scheduler(self): ... + def create_remote_scheduler(self, url): ... + def create_worker(self, scheduler, worker_processes, assistant: bool = ...): ... + +class PidLockAlreadyTakenExit(SystemExit): ... + +def run(*args, **kwargs): ... +def build(tasks, worker_scheduler_factory: Optional[Any] = ..., **env_params): ... diff --git a/stubs/luigi/local_target.pyi b/stubs/luigi/local_target.pyi new file mode 100644 index 0000000..d3883e4 --- /dev/null +++ b/stubs/luigi/local_target.pyi @@ -0,0 +1,35 @@ +# Stubs for luigi.local_target (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional +from luigi.target import FileSystem, FileSystemTarget, AtomicLocalFile + +class atomic_file(AtomicLocalFile): + def move_to_final_destination(self): ... + def generate_tmp_path(self, path): ... + +class LocalFileSystem(FileSystem): + def copy(self, old_path, new_path, raise_if_exists: bool = ...): ... + def exists(self, path): ... + def mkdir(self, path, parents: bool = ..., raise_if_exists: bool = ...): ... + def isdir(self, path): ... + def listdir(self, path): ... + # Signature of "remove" incompatible with supertype "FileSystem" + def remove(self, path, recursive: bool = ...): ... # type: ignore + def move(self, old_path, new_path, raise_if_exists: bool = ...): ... + +class LocalTarget(FileSystemTarget): + fs = ... # type: Any + format = ... # type: Any + is_tmp = ... # type: Any + def __init__(self, path: Optional[Any] = ..., format: Optional[Any] = ..., is_tmp: bool = ...) -> None: ... + def makedirs(self): ... + def open(self, mode: str = ...): ... + def move(self, new_path, raise_if_exists: bool = ...): ... + def move_dir(self, new_path): ... + def remove(self): ... + def copy(self, new_path, raise_if_exists: bool = ...): ... + @property + def fn(self): ... + def __del__(self): ... diff --git a/stubs/luigi/parameter.pyi b/stubs/luigi/parameter.pyi new file mode 100644 index 0000000..a550034 --- /dev/null +++ b/stubs/luigi/parameter.pyi @@ -0,0 +1,131 @@ +# Stubs for luigi.parameter (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from json.encoder import JSONEncoder +from typing import Any, Optional +from collections import Mapping + +class ParameterException(Exception): ... +class MissingParameterException(ParameterException): ... +class UnknownParameterException(ParameterException): ... +class DuplicateParameterException(ParameterException): ... + +class Parameter: + significant = ... # type: Any + positional = ... # type: Any + description = ... # type: Any + always_in_help = ... # type: Any + def __init__(self, default: Any = ..., is_global: bool = ..., significant: bool = ..., description: Optional[Any] = ..., config_path: Optional[Any] = ..., positional: bool = ..., always_in_help: bool = ..., batch_method: Optional[Any] = ...) -> None: ... + def has_task_value(self, task_name, param_name): ... + def task_value(self, task_name, param_name): ... + def parse(self, x): ... + def serialize(self, x): ... + def normalize(self, x): ... + def next_in_enumeration(self, _value): ... + +class _DateParameterBase(Parameter): + interval = ... # type: Any + start = ... # type: Any + def __init__(self, interval: int = ..., start: Optional[Any] = ..., **kwargs) -> None: ... + def date_format(self): ... + def parse(self, s): ... + def serialize(self, dt): ... + +class DateParameter(_DateParameterBase): + date_format = ... # type: str + def next_in_enumeration(self, value): ... + def normalize(self, value): ... + +class MonthParameter(DateParameter): + date_format = ... # type: str + def next_in_enumeration(self, value): ... + def normalize(self, value): ... + +class YearParameter(DateParameter): + date_format = ... # type: str + def next_in_enumeration(self, value): ... + def normalize(self, value): ... + +class _DatetimeParameterBase(Parameter): + interval = ... # type: Any + start = ... # type: Any + def __init__(self, interval: int = ..., start: Optional[Any] = ..., **kwargs) -> None: ... + def date_format(self): ... + def parse(self, s): ... + def serialize(self, dt): ... + def normalize(self, dt): ... + def next_in_enumeration(self, value): ... + +class DateHourParameter(_DatetimeParameterBase): + date_format = ... # type: str + +class DateMinuteParameter(_DatetimeParameterBase): + date_format = ... # type: str + deprecated_date_format = ... # type: str + def parse(self, s): ... + +class DateSecondParameter(_DatetimeParameterBase): + date_format = ... # type: str + +class IntParameter(Parameter): + def parse(self, s): ... + def next_in_enumeration(self, value): ... + +class FloatParameter(Parameter): + def parse(self, s): ... + +class BoolParameter(Parameter): + def __init__(self, *args, **kwargs) -> None: ... + def parse(self, s): ... + def normalize(self, value): ... + +class DateIntervalParameter(Parameter): + def parse(self, s): ... + +class TimeDeltaParameter(Parameter): + def parse(self, input): ... + def serialize(self, x): ... + +class TaskParameter(Parameter): + def parse(self, input): ... + def serialize(self, cls): ... + +class EnumParameter(Parameter): + def __init__(self, *args, **kwargs) -> None: ... + def parse(self, s): ... + def serialize(self, e): ... + +class FrozenOrderedDict(Mapping): + def __init__(self, *args, **kwargs) -> None: ... + def __getitem__(self, key): ... + def __iter__(self): ... + def __len__(self): ... + def __hash__(self): ... + def get_wrapped(self): ... + +class DictParameter(Parameter): + class DictParamEncoder(JSONEncoder): + def default(self, obj): ... + def normalize(self, value): ... + def parse(self, s): ... + def serialize(self, x): ... + +class ListParameter(Parameter): + def normalize(self, x): ... + def parse(self, x): ... + def serialize(self, x): ... + +class TupleParameter(Parameter): + def parse(self, x): ... + def serialize(self, x): ... + +class NumericalParameter(Parameter): + description = ... # type: str + def __init__(self, left_op: Any = ..., right_op: Any = ..., *args, **kwargs) -> None: ... + def parse(self, s): ... + +class ChoiceParameter(Parameter): + description = ... # type: str + def __init__(self, var_type: Any = ..., *args, **kwargs) -> None: ... + def parse(self, s): ... diff --git a/stubs/luigi/rpc.pyi b/stubs/luigi/rpc.pyi new file mode 100644 index 0000000..b73c49f --- /dev/null +++ b/stubs/luigi/rpc.pyi @@ -0,0 +1,26 @@ +# Stubs for luigi.rpc (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional + +HAS_UNIX_SOCKET = ... # type: bool +HAS_REQUESTS = ... # type: bool +logger = ... # type: Any + +class RPCError(Exception): + sub_exception = ... # type: Any + def __init__(self, message, sub_exception: Optional[Any] = ...) -> None: ... + +class URLLibFetcher: + raises = ... # type: Any + def fetch(self, full_url, body, timeout): ... + +class RequestsFetcher: + raises = ... # type: Any + session = ... # type: Any + def __init__(self, session) -> None: ... + def fetch(self, full_url, body, timeout): ... + +class RemoteScheduler: + def __init__(self, url: str = ..., connect_timeout: Optional[Any] = ...) -> None: ... diff --git a/stubs/luigi/target.pyi b/stubs/luigi/target.pyi new file mode 100644 index 0000000..3903aa4 --- /dev/null +++ b/stubs/luigi/target.pyi @@ -0,0 +1,47 @@ +# Stubs for luigi.target (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any +import io + +logger = ... # type: Any + +class Target: + def exists(self) -> bool: ... + +class FileSystemException(Exception): ... +class FileAlreadyExists(FileSystemException): ... +class MissingParentDirectory(FileSystemException): ... +class NotADirectory(FileSystemException): ... + +class FileSystem: + def exists(self, path): ... + def remove(self, path, recursive: bool = ..., skip_trash: bool = ...): ... + def mkdir(self, path, parents: bool = ..., raise_if_exists: bool = ...): ... + def isdir(self, path): ... + def listdir(self, path): ... + def move(self, path, dest): ... + def rename_dont_move(self, path, dest): ... + def rename(self, *args, **kwargs): ... + def copy(self, path, dest): ... + +class FileSystemTarget(Target): + path = ... # type: Any + def __init__(self, path) -> None: ... + def fs(self): ... + def open(self, mode): ... + def exists(self): ... + def remove(self): ... + def temporary_path(self): ... + +class AtomicLocalFile(io.BufferedWriter): + path = ... # type: Any + def __init__(self, path) -> None: ... + def close(self): ... + def generate_tmp_path(self, path): ... + def move_to_final_destination(self): ... + def __del__(self): ... + @property + def tmp_path(self): ... + def __exit__(self, exc_type, exc, traceback): ... diff --git a/stubs/luigi/task.pyi b/stubs/luigi/task.pyi new file mode 100644 index 0000000..66e892c --- /dev/null +++ b/stubs/luigi/task.pyi @@ -0,0 +1,107 @@ +# Stubs for luigi.task (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Dict, Generic, Iterable, List, Optional, TypeVar, Type, Union + +from .target import Target + +Parameter = ... # type: Any +logger = ... # type: Any +TASK_ID_INCLUDE_PARAMS = ... # type: int +TASK_ID_TRUNCATE_PARAMS = ... # type: int +TASK_ID_TRUNCATE_HASH = ... # type: int +TASK_ID_INVALID_CHAR_REGEX = ... # type: Any + +def namespace(namespace: Optional[Any] = ..., scope: str = ...): ... +def auto_namespace(scope: str = ...): ... +def task_id_str(task_family, params): ... + +class BulkCompleteNotImplementedError(NotImplementedError): ... + +_R = TypeVar('_R') +_I = TypeVar('_I') +_O = TypeVar('_O') +_T = TypeVar('_T') + +class Task(Generic[_R, _I, _O]): + priority = ... # type: int + disabled = ... # type: bool + resources = ... # type: Any + worker_timeout = ... # type: Any + max_batch_size = ... # type: Any + @property + def batchable(self): ... + @property + def retry_count(self): ... + @property + def disable_hard_timeout(self): ... + @property + def disable_window_seconds(self): ... + @property + def owner_email(self): ... + @property + def use_cmdline_section(self): ... + @classmethod + def event_handler(cls, event): ... + def trigger_event(self, event, *args, **kwargs): ... + @property + def task_module(self): ... + task_namespace = ... # type: Any + @classmethod + def get_task_namespace(cls): ... + @property + def task_family(self): ... + @classmethod + def get_task_family(cls): ... + @classmethod + def get_params(cls): ... + @classmethod + def batch_param_names(cls): ... + @classmethod + def get_param_names(cls, include_significant: bool = ...) -> List[str]: ... + @classmethod + def get_param_values(cls, params, args, kwargs): ... + param_args = ... # type: Any + param_kwargs = ... # type: Any + task_id = ... # type: str + set_tracking_url = ... # type: Any + set_status_message = ... # type: Any + def __init__(self, *args, **kwargs) -> None: ... + def initialized(self): ... + @classmethod + def from_str_params(cls: Type[_T], params_str: Dict[str, str]) -> _T: ... + def to_str_params(self, only_significant: bool = ...): ... + def clone(self, cls: Optional[Any] = ..., **kwargs): ... + def __hash__(self): ... + def __eq__(self, other): ... + def complete(self) -> bool: ... + @classmethod + def bulk_complete(cls, parameter_tuples): ... + def output(self) -> _O: ... + def requires(self) -> _R: ... + def process_resources(self): ... + def input(self) -> _I: ... + def deps(self): ... + def run(self) -> None: ... + def on_failure(self, exception): ... + def on_success(self): ... + def no_unpicklable_properties(self): ... + +class MixinNaiveBulkComplete: + @classmethod + def bulk_complete(cls, parameter_tuples): ... + +class ExternalTask(Task[_R, _I, _O]): + run = ... # type: Any + +def externalize(taskclass_or_taskobject): ... + +class WrapperTask(Task[_R, _I, _O]): + def complete(self): ... + +class Config(Task[_R, _I, _O]): ... + +def getpaths(struct): ... +def flatten(struct: Union[Dict[Any, _O], Iterable[_O], _O]) -> List[_O]: ... +def flatten_output(task): ... diff --git a/stubs/luigi/tools/__init__.pyi b/stubs/luigi/tools/__init__.pyi new file mode 100644 index 0000000..d516037 --- /dev/null +++ b/stubs/luigi/tools/__init__.pyi @@ -0,0 +1,4 @@ +# Stubs for luigi.tools (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + diff --git a/stubs/luigi/tools/range.pyi b/stubs/luigi/tools/range.pyi new file mode 100644 index 0000000..ad0de09 --- /dev/null +++ b/stubs/luigi/tools/range.pyi @@ -0,0 +1,86 @@ +# Stubs for luigi.tools.range (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any +import luigi + +logger = ... # type: Any + +class RangeEvent(luigi.Event): + COMPLETE_COUNT = ... # type: str + COMPLETE_FRACTION = ... # type: str + DELAY = ... # type: str + +class RangeBase(luigi.WrapperTask): + of = ... # type: Any + of_params = ... # type: Any + start = ... # type: Any + stop = ... # type: Any + reverse = ... # type: Any + task_limit = ... # type: Any + now = ... # type: Any + param_name = ... # type: Any + @property + def of_cls(self): ... + def datetime_to_parameter(self, dt): ... + def parameter_to_datetime(self, p): ... + def datetime_to_parameters(self, dt): ... + def parameters_to_datetime(self, p): ... + def moving_start(self, now): ... + def moving_stop(self, now): ... + def finite_datetimes(self, finite_start, finite_stop): ... + def requires(self): ... + def missing_datetimes(self, finite_datetimes): ... + +class RangeDailyBase(RangeBase): + start = ... # type: Any + stop = ... # type: Any + days_back = ... # type: Any + days_forward = ... # type: Any + def datetime_to_parameter(self, dt): ... + def parameter_to_datetime(self, p): ... + def datetime_to_parameters(self, dt): ... + def parameters_to_datetime(self, p): ... + def moving_start(self, now): ... + def moving_stop(self, now): ... + def finite_datetimes(self, finite_start, finite_stop): ... + +class RangeHourlyBase(RangeBase): + start = ... # type: Any + stop = ... # type: Any + hours_back = ... # type: Any + hours_forward = ... # type: Any + def datetime_to_parameter(self, dt): ... + def parameter_to_datetime(self, p): ... + def datetime_to_parameters(self, dt): ... + def parameters_to_datetime(self, p): ... + def moving_start(self, now): ... + def moving_stop(self, now): ... + def finite_datetimes(self, finite_start, finite_stop): ... + +class RangeByMinutesBase(RangeBase): + start = ... # type: Any + stop = ... # type: Any + minutes_back = ... # type: Any + minutes_forward = ... # type: Any + minutes_interval = ... # type: Any + def datetime_to_parameter(self, dt): ... + def parameter_to_datetime(self, p): ... + def datetime_to_parameters(self, dt): ... + def parameters_to_datetime(self, p): ... + def moving_start(self, now): ... + def moving_stop(self, now): ... + def finite_datetimes(self, finite_start, finite_stop): ... + +def most_common(items): ... +def infer_bulk_complete_from_fs(datetimes, datetime_to_task, datetime_to_re): ... + +class RangeDaily(RangeDailyBase): + def missing_datetimes(self, finite_datetimes): ... + +class RangeHourly(RangeHourlyBase): + def missing_datetimes(self, finite_datetimes): ... + +class RangeByMinutes(RangeByMinutesBase): + def missing_datetimes(self, finite_datetimes): ... diff --git a/stubs/sqlalchemy/__init__.pyi b/stubs/sqlalchemy/__init__.pyi new file mode 100644 index 0000000..8d58e25 --- /dev/null +++ b/stubs/sqlalchemy/__init__.pyi @@ -0,0 +1,124 @@ +# Stubs for sqlalchemy (Python 2) + +from .sql import ( + alias, + and_, + asc, + between, + bindparam, + case, + cast, + collate, + column, + delete, + desc, + distinct, + except_, + except_all, + exists, + extract, + false, + func, + funcfilter, + insert, + intersect, + intersect_all, + join, + literal, + literal_column, + modifier, + not_, + null, + or_, + outerjoin, + outparam, + over, + select, + subquery, + table, + text, + true, + tuple_, + type_coerce, + union, + union_all, + update, +) + +from .types import ( + BIGINT, + BINARY, + BLOB, + BOOLEAN, + BigInteger, + Binary, + Boolean, + CHAR, + CLOB, + DATE, + DATETIME, + DECIMAL, + Date, + DateTime, + Enum, + FLOAT, + Float, + INT, + INTEGER, + Integer, + Interval, + LargeBinary, + NCHAR, + NVARCHAR, + NUMERIC, + Numeric, + PickleType, + REAL, + SMALLINT, + SmallInteger, + String, + TEXT, + TIME, + TIMESTAMP, + Text, + Time, + TypeDecorator, + Unicode, + UnicodeText, + VARBINARY, + VARCHAR, +) + +from .schema import ( + CheckConstraint, + Column, + ColumnDefault, + Constraint, + DefaultClause, + FetchedValue, + ForeignKey, + ForeignKeyConstraint, + Index, + MetaData, + PassiveDefault, + PrimaryKeyConstraint, + Sequence, + Table, + ThreadLocalMetaData, + UniqueConstraint, + DDL, +) + +from . import sql as sql +from . import schema as schema +from . import types as types +from . import exc as exc +from . import dialects as dialects +from . import pool as pool +# This should re-export orm but orm is totally broken right now +# from . import orm as orm + +from .inspection import inspect +from .engine import create_engine, engine_from_config + +__version__ = ... # type: int diff --git a/stubs/sqlalchemy/engine/__init__.pyi b/stubs/sqlalchemy/engine/__init__.pyi new file mode 100644 index 0000000..396cac5 --- /dev/null +++ b/stubs/sqlalchemy/engine/__init__.pyi @@ -0,0 +1,11 @@ +# Stubs for sqlalchemy.engine (Python 2) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from .base import Connection as Connection +from .base import Engine as Engine +from .base import Transaction as Transaction +from .result import RowProxy as RowProxy + +def create_engine(*args, **kwargs): ... +def engine_from_config(configuration, prefix=..., **kwargs): ... diff --git a/stubs/sqlalchemy/engine/base.pyi b/stubs/sqlalchemy/engine/base.pyi new file mode 100644 index 0000000..4f5ff43 --- /dev/null +++ b/stubs/sqlalchemy/engine/base.pyi @@ -0,0 +1,39 @@ +from typing import ( + Any, Generic, List, Optional, Tuple, Type, TypeVar +) +from types import TracebackType + +from .result import ResultProxy + +_T = TypeVar('_T') + +# new in 3.6 +class ContextManager(Generic[_T]): + def __enter__(self) -> _T: ... + def __exit__(self, exc_type: Optional[Type[BaseException]], + exc_val: Optional[Exception], + exc_tb: Optional[TracebackType]) -> bool: ... + + +# Dummy until I figure out something better. +class Connectable: + pass + +class Connection: + def begin(self) -> 'Transaction': ... + def execute(self, object: object, *multiparams: Any, **params: Any) -> ResultProxy: ... + def scalar(self, object: object, *multiparams: Any, **params: Any) -> Any: ... + +class Engine(object): + def execute(self, object: object, *multiparams: Any, **params: Any) -> ResultProxy: ... + def connect(self) -> Connection: ... + +class RowProxy: + def items(self) -> List[Tuple[Any, Any]]: ... + def keys(self) -> List[Any]: ... + def values(self) -> List[Any]: ... + def __getitem__(self, key: str): ... + +class Transaction(ContextManager): + def commit(self): ... + def rollback(self): ... diff --git a/stubs/sqlalchemy/engine/result.pyi b/stubs/sqlalchemy/engine/result.pyi new file mode 100644 index 0000000..a44553b --- /dev/null +++ b/stubs/sqlalchemy/engine/result.pyi @@ -0,0 +1,86 @@ +# Stubs for sqlalchemy.engine.result (Python 3.5) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Iterable, List, Optional +from ..sql import expression as expression, sqltypes as sqltypes +# from ..sql import util as sql_util +# from sqlalchemy.cresultproxy import BaseRowProxy + +def rowproxy_reconstructor(cls, state): ... + +class BaseRowProxy(Iterable): + def __init__(self, parent, row, processors, keymap) -> None: ... + def __reduce__(self): ... + def values(self): ... + def __iter__(self): ... + def __len__(self): ... + def __getitem__(self, key): ... + def __getattr__(self, name): ... + +class RowProxy(BaseRowProxy): + def __contains__(self, key): ... + __hash__ = ... # type: Any + def __lt__(self, other): ... + def __le__(self, other): ... + def __ge__(self, other): ... + def __gt__(self, other): ... + def __eq__(self, other): ... + def __ne__(self, other): ... + def has_key(self, key): ... + def items(self): ... + def keys(self) -> List[str]: ... + def iterkeys(self): ... + def itervalues(self): ... + +class ResultMetaData: + case_sensitive = ... # type: Any + matched_on_name = ... # type: bool + def __init__(self, parent, cursor_description) -> None: ... + +class ResultProxy: + out_parameters = ... # type: Any + closed = ... # type: bool + context = ... # type: Any + dialect = ... # type: Any + cursor = ... # type: Any + connection = ... # type: Any + def __init__(self, context) -> None: ... + def keys(self) -> List[str]: ... + rowcount = ... # type: int + @property + def lastrowid(self): ... + @property + def returns_rows(self): ... + @property + def is_insert(self): ... + def close(self): ... + def __iter__(self): ... + def inserted_primary_key(self): ... + def last_updated_params(self): ... + def last_inserted_params(self): ... + @property + def returned_defaults(self): ... + def lastrow_has_defaults(self): ... + def postfetch_cols(self): ... + def prefetch_cols(self): ... + def supports_sane_rowcount(self): ... + def supports_sane_multi_rowcount(self): ... + def process_rows(self, rows): ... + def fetchall(self) -> List[RowProxy]: ... + def fetchmany(self, size: Optional[Any] = ...): ... + def fetchone(self) -> RowProxy: ... + def first(self): ... + def scalar(self): ... + +class BufferedRowResultProxy(ResultProxy): + size_growth = ... # type: Any + +class FullyBufferedResultProxy(ResultProxy): ... + +class BufferedColumnRow(RowProxy): + def __init__(self, parent, row, processors, keymap) -> None: ... + +class BufferedColumnResultProxy(ResultProxy): + def fetchall(self) -> List[RowProxy]: ... + def fetchmany(self, size: Optional[Any] = ...) -> List[RowProxy]: ... diff --git a/stubs/sqlalchemy/engine/url.pyi b/stubs/sqlalchemy/engine/url.pyi new file mode 100644 index 0000000..7880ccc --- /dev/null +++ b/stubs/sqlalchemy/engine/url.pyi @@ -0,0 +1,27 @@ +# Stubs for sqlalchemy.engine.url (Python 2) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, AnyStr +from .. import dialects + +# registry = dialects.registry + +class URL: + drivername = ... # type: Any + username = ... # type: Any + password = ... # type: Any + host = ... # type: Any + port = ... # type: Any + database = ... # type: Any + query = ... # type: Any + def __init__(self, drivername, username=..., password=..., host=..., port=..., database=..., query=...) -> None: ... + def __to_string__(self, hide_password=...): ... + def __hash__(self): ... + def __eq__(self, other): ... + def get_backend_name(self): ... + def get_driver_name(self): ... + def get_dialect(self): ... + def translate_connect_args(self, names=..., **kw): ... + +def make_url(name_or_url: AnyStr) -> URL: ... diff --git a/stubs/sqlalchemy/exc.pyi b/stubs/sqlalchemy/exc.pyi new file mode 100644 index 0000000..ef3c7af --- /dev/null +++ b/stubs/sqlalchemy/exc.pyi @@ -0,0 +1,27 @@ +# Stubs for sqlalchemy.exc (Python 3.5) +# +# NOTE: This dynamically typed is an excerpt from a +# stub automatically generated by stubgen. + +from typing import Any, Optional + +class SQLAlchemyError(Exception): ... + +class StatementError(SQLAlchemyError): + statement = ... # type: Any + params = ... # type: Any + orig = ... # type: Any + detail = ... # type: Any + def __init__(self, message, statement, params, orig) -> None: ... + def add_detail(self, msg): ... + def __reduce__(self): ... + def __unicode__(self): ... + +class DBAPIError(StatementError): + @classmethod + def instance(cls, statement, params, orig, dbapi_base_err, connection_invalidated: bool = ..., dialect: Optional[Any] = ...): ... + def __reduce__(self): ... + connection_invalidated = ... # type: Any + def __init__(self, statement, params, orig, connection_invalidated: bool = ...) -> None: ... + +class DatabaseError(DBAPIError): ... diff --git a/stubs/sqlalchemy/sql/__init__.pyi b/stubs/sqlalchemy/sql/__init__.pyi new file mode 100644 index 0000000..e69de29 diff --git a/stubs/sqlalchemy/sql/elements.pyi b/stubs/sqlalchemy/sql/elements.pyi new file mode 100644 index 0000000..eabe5b3 --- /dev/null +++ b/stubs/sqlalchemy/sql/elements.pyi @@ -0,0 +1,25 @@ +from typing import Any, Optional + +class Visitable(object): + pass + +class ClauseElement(Visitable): + __visit_name__ = ... # type: str + supports_execution = ... # type: bool + bind = ... # type: Any + is_selectable = ... # type: bool + is_clause_element = ... # type: bool + description = ... # type: Any + def unique_params(self, *optionaldict, **kwargs): ... + def params(self, *optionaldict, **kwargs): ... + def compare(self, other, **kw): ... + def get_children(self, **kwargs): ... + def self_group(self, against: Optional[Any] = ...): ... + # @util.dependencies("sqlalchemy.engine.default") + # def compile(self, default, bind: Optional[Any] = ..., dialect: Optional[Any] = ..., **kw): ... + def compile(self, bind: Optional[Any] = ..., dialect: Optional[Any] = ..., **kw): ... + def __and__(self, other): ... + def __or__(self, other): ... + def __invert__(self): ... + def __bool__(self): ... + __nonzero__ = ... # type: Any diff --git a/stubs/sqlalchemy/sql/expression.pyi b/stubs/sqlalchemy/sql/expression.pyi new file mode 100644 index 0000000..f361bd2 --- /dev/null +++ b/stubs/sqlalchemy/sql/expression.pyi @@ -0,0 +1,7 @@ +from .elements import ClauseElement + +class Select(ClauseElement): + @property + def columns(self): ... + c = ... # type: Any + def where(self, whereclause): ...