diff --git a/Oracle/PCORI_MEDS_SCHEMA_CHANGE_ora.sql b/Oracle/PCORI_MEDS_SCHEMA_CHANGE_ora.sql deleted file mode 100644 index 139f479..0000000 --- a/Oracle/PCORI_MEDS_SCHEMA_CHANGE_ora.sql +++ /dev/null @@ -1,50 +0,0 @@ -alter table pcornet_med add PCORI_NDC varchar2(12); -UPDATE PCORNET_MED set pcori_ndc = pcori_basecode where length(pcori_basecode)=11 and c_hlevel>2 and LOWER(sourcesystem_cd) not in ('integration_tool') -and pcori_basecode not like 'N%'; - -alter table pcornet_med add PCORI_CUI varchar2(8); -UPDATE pcornet_med set PCORI_CUI = PCORI_BASECODE where length(pcori_basecode)<11 and c_hlevel>2 and LOWER(sourcesystem_cd) not in ('integration_tool') -and pcori_basecode not like 'N%' and m_applied_path='@'; - - -create table CUI_T as -select * from -( -with -cui(c_fullname,pcori_cui,c_hlevel) as -( - select c_fullname,pcori_cui,c_hlevel from pcornet_med where pcori_cui is not null - union all - select m.c_fullname, cui.pcori_cui, m.c_hlevel - from pcornet_med m inner join cui on cui.c_fullname=m.c_path where m.pcori_cui is null -), -cuid as -( select c_fullname, pcori_cui, row_number() over (partition by C_FULLNAME order by c_hlevel desc)as rowno from cui) - -select DISTINCT cuid.c_fullname as c_fullname, cuid.pcori_cui as pcori_cui from cuid where cuid.rowno=1 -); - -update pcornet_med x set x.pcori_cui = - (select pcori_cui from CUI_T t where x.c_fullname = t.c_fullname) -where x.pcori_cui is null; - - - -create table NDC_T as -select * from -( -with ndc(c_fullname,pcori_ndc,c_hlevel) as -( - select c_fullname,pcori_ndc,c_hlevel from pcornet_med where pcori_ndc is not null - union all - select m.c_fullname,ndc.pcori_ndc,m.c_hlevel from pcornet_med m - inner join ndc on ndc.c_fullname=m.c_path where m.pcori_ndc is null -), -ndcd as ( select c_fullname,pcori_ndc, row_number() over (partition by C_FULLNAME order by c_hlevel desc) row from ndc) - -select DISTINCT ndcd.c_fullname as c_fullname, ndcd.pcori_ndc as pcori_ndc from ndcd where ndcd.row=1 -); - -update pcornet_med x set x.pcori_ndc = - (select pcori_ndc from NDC_T t where x.c_fullname = t.c_fullname) -where x.pcori_ndc is null; \ No newline at end of file diff --git a/Oracle/PCORNetInit.sql b/Oracle/PCORNetInit.sql new file mode 100644 index 0000000..0e96fb7 --- /dev/null +++ b/Oracle/PCORNetInit.sql @@ -0,0 +1,987 @@ +------------------------------------------------------------------------------------------- +-- PCORNetInit Script +-- +-- This script prepares target load and intermidiary transform tables, helper +-- functions and procedure, and synonyms and views of the source i2b2 source +-- tables. +-- +-- This script should be run as the initial step of running the i2p-transform, +-- but has been seperated out from PCORNetLoader_ora.sql so that the former can +-- be run without dropping all tables. +-- +-- Created by: Michael Prittie (mprittie@kumc.edu) +-- Adapted from original PCORNetLoader_ora.sql +-------------------------------------------------------------------------------- + + +-------------------------------------------------------------------------------- +-- HELPER FUNCTIONS AND PROCEDURES +-------------------------------------------------------------------------------- + +create or replace PROCEDURE GATHER_TABLE_STATS(table_name VARCHAR2) AS + BEGIN + DBMS_STATS.GATHER_TABLE_STATS ( + ownname => 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. + tabname => table_name, + estimate_percent => 50, -- Percentage picked somewhat arbitrarily + cascade => TRUE, + degree => 16 + ); +END GATHER_TABLE_STATS; +/ + + +create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS + BEGIN + EXECUTE IMMEDIATE sqlstring; + EXCEPTION + WHEN OTHERS THEN NULL; +END PMN_DROPSQL; +/ + + +create or replace FUNCTION PMN_IFEXISTS(objnamestr VARCHAR2, objtypestr VARCHAR2) RETURN BOOLEAN AS +cnt NUMBER; +BEGIN + SELECT COUNT(*) + INTO cnt + FROM USER_OBJECTS + WHERE upper(OBJECT_NAME) = upper(objnamestr) + and upper(object_type) = upper(objtypestr); + + IF( cnt = 0 ) + THEN + --dbms_output.put_line('NO!'); + return FALSE; + ELSE + --dbms_output.put_line('YES!'); + return TRUE; + END IF; + +END PMN_IFEXISTS; +/ + + +create or replace PROCEDURE PMN_Execuatesql(sqlstring VARCHAR2) AS +BEGIN + EXECUTE IMMEDIATE sqlstring; + dbms_output.put_line(sqlstring); +END PMN_ExecuateSQL; +/ + + +--ACK: http://dba.stackexchange.com/questions/9441/how-to-catch-and-handle-only-specific-oracle-exceptions +create or replace procedure create_error_table(table_name varchar2) as +sqltext varchar2(4000); + +begin + dbms_errlog.create_error_log(dml_table_name => table_name); +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -955 THEN + NULL; -- suppresses ORA-00955 exception ("name is already used by an existing object") + ELSE + RAISE; + END IF; +-- Delete rows from a previous run in case the table already existed +sqltext := 'delete from ERR$_' || table_name; +PMN_Execuatesql(sqltext); +end; +/ + + +create or replace FUNCTION GETDATAMARTID RETURN VARCHAR2 IS +BEGIN + RETURN '&&datamart_id'; +END; +/ + + +CREATE OR REPLACE FUNCTION GETDATAMARTNAME RETURN VARCHAR2 AS +BEGIN + RETURN '&&datamart_name'; +END; +/ + + +CREATE OR REPLACE FUNCTION GETDATAMARTPLATFORM RETURN VARCHAR2 AS +BEGIN + RETURN '02'; -- 01 is MSSQL, 02 is Oracle +END; +/ + + +BEGIN +PMN_DROPSQL('DROP TABLE pcornet_codelist'); +END; +/ +create table pcornet_codelist(codetype varchar2(20), code varchar2(50)) +/ + +create or replace procedure pcornet_parsecode (codetype in varchar, codestring in varchar) as + +tex varchar(2000); +pos number(9); +readstate char(1) ; +nextchar char(1) ; +val varchar(50); + +begin + +val:=''; +readstate:='F'; +pos:=0; +tex := codestring; +FOR pos IN 1..length(tex) +LOOP +-- dbms_output.put_line(val); + nextchar:=substr(tex,pos,1); + if nextchar!=',' then + if nextchar='''' then + if readstate='F' then + val:=''; + readstate:='T'; + else + insert into pcornet_codelist values (codetype,val); + val:=''; + readstate:='F' ; + end if; + else + if readstate='T' then + val:= val || nextchar; + end if; + end if; + end if; +END LOOP; + +end pcornet_parsecode; +/ + +create or replace procedure pcornet_popcodelist as + +codedata varchar(2000); +onecode varchar(20); +codetype varchar(20); + +cursor getcodesql is +select 'RACE',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\RACE%' +union +select 'SEX',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\SEX%' +union +select 'HISPANIC',c_dimcode from pcornet_demo where c_fullname like '\PCORI\DEMOGRAPHIC\HISPANIC%'; + +begin +open getcodesql; +LOOP + fetch getcodesql into codetype,codedata; + EXIT WHEN getcodesql%NOTFOUND ; + pcornet_parsecode (codetype,codedata ); +end loop; + +close getcodesql ; +end pcornet_popcodelist; +/ + + +-------------------------------------------------------------------------------- +-- I2B2 SYNONYMS, VIEWS, AND INTERMEDIARY TABLES +-------------------------------------------------------------------------------- + + +CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT +/ + +CREATE OR REPLACE SYNONYM I2B2MEDFACT FOR OBSERVATION_FACT_MEDS +/ + +BEGIN +PMN_DROPSQL('DROP TABLE i2b2patient_list'); +END; +/ + +CREATE table i2b2patient_list as +select * from +( +select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') +) where ROWNUM<100000000 +/ + +create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) +/ + +create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE 'PCORNET_CDM', -- This doesn't work as a parameter for some reason. - tabname => table_name, - estimate_percent => 50, -- Percentage picked somewhat arbitrarily - cascade => TRUE, - degree => 16 - ); -END GATHER_TABLE_STATS; -/ - -create or replace PROCEDURE PMN_DROPSQL(sqlstring VARCHAR2) AS - BEGIN - EXECUTE IMMEDIATE sqlstring; - EXCEPTION - WHEN OTHERS THEN NULL; -END PMN_DROPSQL; -/ - -create or replace FUNCTION PMN_IFEXISTS(objnamestr VARCHAR2, objtypestr VARCHAR2) RETURN BOOLEAN AS -cnt NUMBER; -BEGIN - SELECT COUNT(*) - INTO cnt - FROM USER_OBJECTS - WHERE upper(OBJECT_NAME) = upper(objnamestr) - and upper(object_type) = upper(objtypestr); - - IF( cnt = 0 ) - THEN - --dbms_output.put_line('NO!'); - return FALSE; - ELSE - --dbms_output.put_line('YES!'); - return TRUE; - END IF; - -END PMN_IFEXISTS; -/ - - -create or replace PROCEDURE PMN_Execuatesql(sqlstring VARCHAR2) AS -BEGIN - EXECUTE IMMEDIATE sqlstring; - dbms_output.put_line(sqlstring); -END PMN_ExecuateSQL; -/ - ---ACK: http://dba.stackexchange.com/questions/9441/how-to-catch-and-handle-only-specific-oracle-exceptions -create or replace procedure create_error_table(table_name varchar2) as -sqltext varchar2(4000); - -begin - dbms_errlog.create_error_log(dml_table_name => table_name); -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE = -955 THEN - NULL; -- suppresses ORA-00955 exception ("name is already used by an existing object") - ELSE - RAISE; - END IF; --- Delete rows from a previous run in case the table already existed -sqltext := 'delete from ERR$_' || table_name; -PMN_Execuatesql(sqltext); -end; -/ - - - - - - -CREATE OR REPLACE SYNONYM I2B2FACT FOR "&&i2b2_data_schema".OBSERVATION_FACT -/ - -CREATE OR REPLACE SYNONYM I2B2MEDFACT FOR OBSERVATION_FACT_MEDS -/ - -BEGIN -PMN_DROPSQL('DROP TABLE i2b2patient_list'); -END; -/ - -CREATE table i2b2patient_list as -select * from -( -select DISTINCT PATIENT_NUM from I2B2FACT where START_DATE > to_date('&&min_pat_list_date_dd_mon_rrrr','dd-mon-rrrr') -) where ROWNUM<100000000 -/ - -create or replace VIEW i2b2patient as select * from "&&i2b2_data_schema".PATIENT_DIMENSION where PATIENT_NUM in (select PATIENT_NUM from i2b2patient_list) -/ - -create or replace view i2b2visit as select * from "&&i2b2_data_schema".VISIT_DIMENSION where START_DATE >= to_date('&&min_visit_date_dd_mon_rrrr','dd-mon-rrrr') and (END_DATE is NULL or END_DATE < CURRENT_DATE) and (START_DATE 0 then current_date else null end REFRESH_DEMOGRAPHIC_DATE, @@ -1495,137 +719,70 @@ end PCORNetHarvest; -/* TODO: When compiling PCORNetPrescribing, I got Error(93,15): - PL/SQL: ORA-00942: table or view does not exist -At compile time, it's complaining about the fact tables don't exist that are -created in the function itself. I created them ahead of time - SQL taken from -the procedure. -*/ -whenever sqlerror continue; -drop table basis; -drop table freq; -drop table quantity; -drop table refills; -drop table supply; - -create table basis ( - pcori_basecode varchar2(50 byte), - c_fullname varchar2(700 byte), - encounter_num number(38,0), - concept_cd varchar2(50 byte), - instance_num number(18,0), - start_date date, - provider_id varchar2(50 byte), - modifier_cd varchar2(100 byte) - ) ; - -create table freq ( - pcori_basecode varchar2(50 byte), - encounter_num number(38,0), - concept_cd varchar2(50 byte), - instance_num number(18,0), - start_date date, - provider_id varchar2(50 byte), - modifier_cd varchar2(100 byte) - ); - -create table quantity( - nval_num number(18,5), - encounter_num number(38,0), - concept_cd varchar2(50 byte), - instance_num number(18,0), - start_date date, - provider_id varchar2(50 byte), - modifier_cd varchar2(100 byte) - ); - -create table refills( - nval_num number(18,5), - encounter_num number(38,0), - concept_cd varchar2(50 byte), - instance_num number(18,0), - start_date date, - provider_id varchar2(50 byte), - modifier_cd varchar2(100 byte) - ); - -create table supply( - nval_num number(18,5), - encounter_num number(38,0), - concept_cd varchar2(50 byte), - instance_num number(18,0), - start_date date, - provider_id varchar2(50 byte), - modifier_cd varchar2(100 byte) - ); - -whenever sqlerror exit; create or replace procedure PCORNetPrescribing as -sqltext varchar2(4000); begin -PMN_DROPSQL('drop index prescribing_patid'); -PMN_DROPSQL('drop index prescribing_encounterid'); - - -PMN_DROPSQL('DROP TABLE basis'); -sqltext := 'create table basis as '|| -'(select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact basis '|| -' inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num '|| -' join pcornet_med basiscode '|| -' on basis.modifier_cd = basiscode.c_basecode '|| -' and basiscode.c_fullname like ''\PCORI_MOD\RX_BASIS\%'') '; -PMN_EXECUATESQL(sqltext); +PMN_DROPSQL('drop index prescribing_idx'); +PMN_DROPSQL('drop index basis_idx'); +PMN_DROPSQL('drop index freq_idx'); +PMN_DROPSQL('drop index quantity_idx'); +PMN_DROPSQL('drop index refills_idx'); +PMN_DROPSQL('drop index supply_idx'); + +execute immediate 'truncate table prescribing'; +execute immediate 'truncate table basis'; +execute immediate 'truncate table freq'; +execute immediate 'truncate table quantity'; +execute immediate 'truncate table refills'; +execute immediate 'truncate table supply'; + +insert into table basis +select pcori_basecode,c_fullname,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact basis + inner join encounter enc on enc.patid = basis.patient_num and enc.encounterid = basis.encounter_Num + join pcornet_med basiscode + on basis.modifier_cd = basiscode.c_basecode + and basiscode.c_fullname like '\PCORI_MOD\RX_BASIS\%'; execute immediate 'create unique index basis_idx on basis (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('BASIS'); -PMN_DROPSQL('DROP TABLE freq'); -sqltext := 'create table freq as '|| -'(select pcori_basecode,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact freq '|| -' inner join encounter enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num '|| -' join pcornet_med freqcode '|| -' on freq.modifier_cd = freqcode.c_basecode '|| -' and freqcode.c_fullname like ''\PCORI_MOD\RX_FREQUENCY\%'') '; -PMN_EXECUATESQL(sqltext); +insert into table freq +select pcori_basecode,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact freq + inner join encounter enc on enc.patid = freq.patient_num and enc.encounterid = freq.encounter_Num + join pcornet_med freqcode + on freq.modifier_cd = freqcode.c_basecode + and freqcode.c_fullname like '\PCORI_MOD\RX_FREQUENCY\%'; execute immediate 'create unique index freq_idx on freq (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('FREQ'); -PMN_DROPSQL('DROP TABLE quantity'); -sqltext := 'create table quantity as '|| -'(select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact quantity '|| -' inner join encounter enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num '|| -' join pcornet_med quantitycode '|| -' on quantity.modifier_cd = quantitycode.c_basecode '|| -' and quantitycode.c_fullname like ''\PCORI_MOD\RX_QUANTITY\'') '; - -PMN_EXECUATESQL(sqltext); +insert into table quantity +select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact quantity + inner join encounter enc on enc.patid = quantity.patient_num and enc.encounterid = quantity.encounter_Num + join pcornet_med quantitycode + on quantity.modifier_cd = quantitycode.c_basecode + and quantitycode.c_fullname like '\PCORI_MOD\RX_QUANTITY\'; execute immediate 'create unique index quantity_idx on quantity (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('QUANTITY'); -PMN_DROPSQL('DROP TABLE refills'); -sqltext := 'create table refills as '|| -'(select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact refills '|| -' inner join encounter enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num '|| -' join pcornet_med refillscode '|| -' on refills.modifier_cd = refillscode.c_basecode '|| -' and refillscode.c_fullname like ''\PCORI_MOD\RX_REFILLS\'') '; -PMN_EXECUATESQL(sqltext); +insert into table refills +select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact refills + inner join encounter enc on enc.patid = refills.patient_num and enc.encounterid = refills.encounter_Num + join pcornet_med refillscode + on refills.modifier_cd = refillscode.c_basecode + and refillscode.c_fullname like '\PCORI_MOD\RX_REFILLS\'; execute immediate 'create unique index refills_idx on refills (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('REFILLS'); -PMN_DROPSQL('DROP TABLE supply'); -sqltext := 'create table supply as '|| -'(select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact supply '|| -' inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num '|| -' join pcornet_med supplycode '|| -' on supply.modifier_cd = supplycode.c_basecode '|| -' and supplycode.c_fullname like ''\PCORI_MOD\RX_DAYS_SUPPLY\'') '; -PMN_EXECUATESQL(sqltext); +insert into table supply +select nval_num,instance_num,start_date,provider_id,concept_cd,encounter_num,modifier_cd from i2b2medfact supply + inner join encounter enc on enc.patid = supply.patient_num and enc.encounterid = supply.encounter_Num + join pcornet_med supplycode + on supply.modifier_cd = supplycode.c_basecode + and supplycode.c_fullname like '\PCORI_MOD\RX_DAYS_SUPPLY\'; execute immediate 'create unique index supply_idx on supply (instance_num, start_date, provider_id, concept_cd, encounter_num, modifier_cd)'; GATHER_TABLE_STATS('SUPPLY'); @@ -1694,8 +851,7 @@ inner join encounter enc on enc.encounterid = m.encounter_Num where (basis.c_fullname is null or basis.c_fullname like '\PCORI_MOD\RX_BASIS\PR\%'); -execute immediate 'create index prescribing_patid on prescribing (PATID)'; -execute immediate 'create index prescribing_encounterid on prescribing (ENCOUNTERID)'; +execute immediate 'create index prescribing_idx on prescribing (PATID, ENCOUNTERID)'; GATHER_TABLE_STATS('PRESCRIBING'); end PCORNetPrescribing; @@ -1703,33 +859,14 @@ end PCORNetPrescribing; -/* TODO: When compiling PCORNetDispensing: - -Error(53,16): PL/SQL: ORA-00942: table or view does not exist (amount) - - supply, also used, is created above in the prescribing function - -Also, Error(57,57): PL/SQL: ORA-00904: "MO"."PCORI_NDC": invalid identifier -*/ -whenever sqlerror continue; -drop table amount; - -create table amount( - nval_num number(18,5), - encounter_num number(38,0), - concept_cd varchar2(50 byte) - ); - -alter table "&&i2b2_meta_schema".pcornet_med add ( - pcori_ndc varchar2(1000) -- arbitrary - ); -whenever sqlerror exit; create or replace procedure PCORNetDispensing as -sqltext varchar2(4000); begin -/* -PMN_DROPSQL('drop index dispensing_patid'); +PMN_DROPSQL('drop index dispensing_idx'); + +execute immediate 'truncate table dispensing'; +/* PMN_DROPSQL('DROP TABLE supply'); sqltext := 'create table supply as '|| '(select nval_num,encounter_num,concept_cd from i2b2fact supply '|| @@ -1838,7 +975,8 @@ inner join encounter enc on enc.encounterid = m.encounter_Num group by m.encounter_num ,m.patient_num, m.start_date, mo.pcori_ndc; */ -execute immediate 'create index dispensing_patid on dispensing (PATID)'; +execute immediate 'create index dispensing_idx on dispensing (PATID)'; +GATHER_TABLE_STATS('DISPENSING'); end PCORNetDispensing; / @@ -1854,8 +992,9 @@ end PCORNetDispensing; create or replace procedure PCORNetDeath as - begin + +execute immediate 'truncate table death'; insert into death( patid, death_date, death_date_impute, death_source, death_match_confidence) select distinct pat.patient_num, pat.death_date, @@ -1885,116 +1024,149 @@ end; / -create or replace PROCEDURE pcornetReport -as -i2b2pats number; -i2b2Encounters number; -i2b2facts number; -i2b2dxs number; -i2b2procs number; -i2b2lcs number; - -pmnpats number; -encounters number; -pmndx number; -pmnprocs number; -pmnfacts number; -pmnenroll number; -vital number; - - - -pmnlabs number; -prescribings number; -dispensings number; -pmncond number; - - -v_runid number; +-------------------------------------------------------------------------------- +-- PCORNetPostProc procedure +-- +-- Ideally this procedure would be empty, but as a matter of practice there are +-- often post processing cleanup tasks that needs to be complete. Such tasks +-- should reside here with the goal of incorporating them into to the proper +-- procedures above. +-------------------------------------------------------------------------------- +create or replace procedure PCORNetPostProc as begin -select count(*) into i2b2Pats from i2b2patient; -select count(*) into i2b2Encounters from i2b2visit i inner join demographic d on i.patient_num=d.patid; - - -select count(*) into pmnPats from demographic; -select count(*) into encounters from encounter e ; -select count(*) into pmndx from diagnosis; -select count(*) into pmnprocs from procedures; - -select count(*) into pmncond from condition; -select count(*) into pmnenroll from enrollment; -select count(*) into vital from vital; -select count(*) into pmnlabs from lab_result_cm; -select count(*) into prescribings from prescribing; -select count(*) into dispensings from dispensing; - -select max(runid) into v_runid from i2pReport; -v_runid := v_runid + 1; -insert into i2pReport values( v_runid, SYSDATE, 'Pats', i2b2pats, pmnpats, i2b2pats-pmnpats); -insert into i2pReport values( v_runid, SYSDATE, 'Enrollment', i2b2pats, pmnenroll, i2b2pats-pmnpats); - -insert into i2pReport values(v_runid, SYSDATE, 'Encounters', i2b2Encounters, encounters, i2b2encounters-encounters); -insert into i2pReport values(v_runid, SYSDATE, 'DX', null, pmndx, null); -insert into i2pReport values(v_runid, SYSDATE, 'PX', null, pmnprocs, null); -insert into i2pReport values(v_runid, SYSDATE, 'Condition', null, pmncond, null); -insert into i2pReport values(v_runid, SYSDATE, 'Vital', null, vital, null); -insert into i2pReport values(v_runid, SYSDATE, 'Labs', null, pmnlabs, null); -insert into i2pReport values(v_runid, SYSDATE, 'Prescribing', null, prescribings,null); -insert into i2pReport values(v_runid, SYSDATE, 'Dispensing', null, dispensings, null); - -end pcornetReport; + + /* Copy providerid from encounter table to diagnosis, procedures tables. + CDM specification says: + "Please note: This is a field replicated from the ENCOUNTER table." + */ + merge into diagnosis d + using encounter e + on (d.encounterid = e.encounterid) + when matched then update set d.providerid = e.providerid; + + merge into procedures p + using encounter e + on (p.encounterid = e.encounterid) + when matched then update set p.providerid = e.providerid; + + merge into prescribing p + using encounter e + on (p.encounterid = e.encounterid) + when matched then update set p.rx_providerid = e.providerid; + + /* Currently in HERON, we have hight in cm and weight in oz (from visit vitals). + The CDM wants height in inches and weight in pounds. */ + update vital v set v.ht = v.ht / 2.54; + update vital v set v.wt = v.wt / 16; + + /* Remove rows from the PRESCRIBING table where RX_* fields are null + TODO: Remove this when fixed in HERON + */ + delete + from prescribing + where rx_basis is null + and rx_quantity is null + and rx_frequency is null + and rx_refills is null + ; + +end PCORNetPostProc; / - - -create or replace procedure pcornetloader as +-------------------------------------------------------------------------------- +-- PCORNetLoader procedure +-- +-- This procedure orchestrates the execution of the procedures defined above, +-- and consists of 13 steps. Using the start_with_step parameter a step number +-- can be provided to start at any point in the sequence. This is helpful when +-- an issue is encountered during execution and restart from the beginning is +-- undesirable. +-- +-- Steps: +-- 1 - PCORNetDemographic +-- 2 - PCORNetEncounter +-- 3 - PCORNetDiagnosis +-- 4 - PCORNetCondition +-- 5 - PCORNetProcedure +-- 6 - PCORNetVital +-- 7 - PCORNetEnroll +-- 8 - PCORNetLabResultCM +-- 9 - PCORNetPrescribing +-- 10 - PCORNetDispensing +-- 11 - PCORNetDeath +-- 12 - PCORNetHarvest +-- 13 - PCORNetPostProc +-- +-------------------------------------------------------------------------------- +create or replace PROCEDURE PCORNetLoader(start_with_step VARCHAR2) AS +start_with_step int; begin ----pcornetclear; -PCORNetDemographic; -PCORNetEncounter; -PCORNetDiagnosis; -PCORNetCondition; -PCORNetProcedure; -PCORNetVital; -PCORNetEnroll; -PCORNetLabResultCM; -PCORNetPrescribing; - -/* ORA-04068: existing state of packages has been discarded -ORA-04065: not executed, altered or dropped stored procedure "PCORNETDISPENSING" -ORA-06508: PL/SQL: could not find program unit being called: "PCORNETDISPENSING" -ORA-06512: at "PCORNETLOADER", line 14 -ORA-06512: at line 2 -04068. 00000 - "existing state of packages%s%s%s has been discarded" -*Cause: One of errors 4060 - 4067 when attempt to execute a stored - procedure. -*Action: Try again after proper re-initialization of any application's - state. - -The above error only happens when we call PCORNetDispensing _and_ PCORNetPrescribing -from within pcornetloader. When running either individually, the error does not -happen. - -Skipping dispensing as per gpc-dev notes: -http://listserv.kumc.edu/pipermail/gpc-dev/attachments/20160223/8d79fa70/attachment-0001.pdf -> LV: the dispensing side [?] is not mandatory? we just did Rx, since that -> what we have in our i2b2 -*/ -PCORNetDispensing; -PCORNetDeath; -PCORNetHarvest; - -end pcornetloader; -/ - - -BEGIN -pcornetloader; --- you may want to run sql statements one by one in the pcornetloader procedure :) -END; -/ - -select concept "Data Type",sourceval "From i2b2",destval "In PopMedNet", diff "Difference" from i2preport where RUNID = (select max(RUNID) from I2PREPORT); - - - + + select step into start_with_step from ( + select 1 step, 'PCORNetDemographic' proc from dual union + select 2 step, 'PCORNetEncounter' proc from dual union + select 3 step, 'PCORNetDiagnosis' proc from dual union + select 4 step, 'PCORNetCondition' proc from dual union + select 5 step, 'PCORNetProcedure' proc from dual union + select 6 step, 'PCORNetVital' proc from dual union + select 7 step, 'PCORNetEnroll' proc from dual union + select 8 step, 'PCORNetLabResultCM' proc from dual union + select 9 step, 'PCORNetPrescribing' proc from dual union + select 10 step, 'PCORNetDispensing' proc from dual union + select 11 step, 'PCORNetDeath' proc from dual union + select 12 step, 'PCORNetHarvest' proc from dual union + select 13 step, 'PCORNetPostProc' proc from dual + ) where proc=start_with; + + if start_with_step = 1 then + PCORNetDemographic; + end if; + + if start_with_step >= 2 then + PCORNetEncounter; + end if; + + if start_with_step >= 3 then + PCORNetDiagnosis; + end if; + + if start_with_step >= 4 then + PCORNetCondition; + end if; + + if start_with_step >= 5 then + PCORNetProcedure; + end if; + + if start_with_step >= 6 then + PCORNetVital; + end if; + + if start_with_step >= 7 then + PCORNetEnroll; + end if; + + if start_with _step >= 8 then + PCORNetLabResultCM; + end if; + + if start_with_step >= 9 then + PCORNetPrescribing; + end if; + + if start_with_step >= 10 then + PCORNetDispensing; + end if; + + if start_with_step >= 11 then + PCORNetDeath; + end if; + + if start_with_step >= 12 then + PCORNetHarvest; + end if; + + PCORNetPostProc; +end PCORNetLoader; +/ \ No newline at end of file diff --git a/Oracle/cdm_postproc.sql b/Oracle/cdm_postproc.sql deleted file mode 100644 index 54a26d2..0000000 --- a/Oracle/cdm_postproc.sql +++ /dev/null @@ -1,37 +0,0 @@ -/* Post-processing tasks against the CDM -*/ - -/* Copy providerid from encounter table to diagnosis, procedures tables. -CDM specification says: - "Please note: This is a field replicated from the ENCOUNTER table." -*/ -merge into diagnosis d -using encounter e - on (d.encounterid = e.encounterid) -when matched then update set d.providerid = e.providerid; - -merge into procedures p -using encounter e - on (p.encounterid = e.encounterid) -when matched then update set p.providerid = e.providerid; - -merge into prescribing p -using encounter e - on (p.encounterid = e.encounterid) -when matched then update set p.rx_providerid = e.providerid; - -/* Currently in HERON, we have hight in cm and weight in oz (from visit vitals). -The CDM wants height in inches and weight in pounds. */ -update vital v set v.ht = v.ht / 2.54; -update vital v set v.wt = v.wt / 16; - -/* Remove rows from the PRESCRIBING table where RX_* fields are null - TODO: Remove this when fixed in HERON - */ -delete -from pcornet_cdm.prescribing -where rx_basis is null - and rx_quantity is null - and rx_frequency is null - and rx_refills is null -; \ No newline at end of file diff --git a/Oracle/gather_table_stats.sql b/Oracle/gather_table_stats.sql deleted file mode 100644 index 94f1a5e..0000000 --- a/Oracle/gather_table_stats.sql +++ /dev/null @@ -1,20 +0,0 @@ -/* Metadata table statistics - experimentation shows vastly better performance -if this is done before the CDM transform. - -ACK: http://stackoverflow.com/questions/2242024/for-each-string-execute-a-function-procedure -*/ -declare -table_list sys.dbms_debug_vc2coll := sys.dbms_debug_vc2coll( - 'PCORNET_DEMO', 'PCORNET_DIAG', 'PCORNET_ENC', 'PCORNET_ENROLL', 'PCORNET_LAB', - 'PCORNET_MED', 'PCORNET_PROC', 'PCORNET_VITAL'); -begin - for t in table_list.first .. table_list.last - loop - DBMS_STATS.GATHER_TABLE_STATS ( - ownname => '"&&i2b2_meta_schema"', - tabname => table_list(t), - estimate_percent => 40 - ); - end loop; -end; -/ diff --git a/Oracle/lab_loinc_mapping.csv b/Oracle/lab_loinc_mapping.csv deleted file mode 100644 index 9338321..0000000 --- a/Oracle/lab_loinc_mapping.csv +++ /dev/null @@ -1,156 +0,0 @@ -"LOINC_CODE","LAB_NAME","PCORI_SPECIMEN_SOURCE" -"13457-7","LDL","" -"18261-8","LDL","" -"2089-1","LDL","" -"47213-4","LDL","SERUM, PLASMA, or SR_PLS" -"47213-4","LDL","SERUM, PLASMA, or SR_PLS" -"47213-4","LDL","SERUM, PLASMA, or SR_PLS" -"49132-4","LDL","" -"17855-8","A1C","" -"17856-6","A1C","BLOOD" -"4548-4","A1C","BLOOD" -"59261-8","A1C","" -"62388-4","A1C","" -"2157-6","CK","SERUM, PLASMA, or SR_PLS" -"2157-6","CK","SERUM, PLASMA, or SR_PLS" -"2157-6","CK","SERUM, PLASMA, or SR_PLS" -"2157-6","CK","SERUM, PLASMA, or SR_PLS" -"50756-6","CK","BLOOD" -"50756-6","CK","BLOOD" -"50756-6","CK","BLOOD" -"50756-6","CK","BLOOD" -"13969-1","CK_MB","SERUM, PLASMA, or SR_PLS" -"13969-1","CK_MB","SERUM, PLASMA, or SR_PLS" -"13969-1","CK_MB","SERUM, PLASMA, or SR_PLS" -"13969-1","CK_MB","SERUM, PLASMA, or SR_PLS" -"2154-3","CK_MB","SERUM, PLASMA, or SR_PLS" -"2154-3","CK_MB","SERUM, PLASMA, or SR_PLS" -"2154-3","CK_MB","SERUM, PLASMA, or SR_PLS" -"2154-3","CK_MB","SERUM, PLASMA, or SR_PLS" -"32673-6","CK_MB","SERUM, PLASMA, or SR_PLS" -"32673-6","CK_MB","SERUM, PLASMA, or SR_PLS" -"32673-6","CK_MB","SERUM, PLASMA, or SR_PLS" -"32673-6","CK_MB","SERUM, PLASMA, or SR_PLS" -"49551-5","CK_MB","BLOOD" -"49551-5","CK_MB","BLOOD" -"49551-5","CK_MB","BLOOD" -"49551-5","CK_MB","BLOOD" -"12187-1","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12187-1","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12187-1","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12187-1","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12189-7","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12189-7","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12189-7","CK_MBI","SERUM, PLASMA, or SR_PLS" -"12189-7","CK_MBI","SERUM, PLASMA, or SR_PLS" -"20569-0","CK_MBI","SERUM, PLASMA, or SR_PLS" -"20569-0","CK_MBI","SERUM, PLASMA, or SR_PLS" -"20569-0","CK_MBI","SERUM, PLASMA, or SR_PLS" -"20569-0","CK_MBI","SERUM, PLASMA, or SR_PLS" -"49136-5","CK_MBI","SERUM, PLASMA, or SR_PLS" -"49136-5","CK_MBI","SERUM, PLASMA, or SR_PLS" -"49136-5","CK_MBI","SERUM, PLASMA, or SR_PLS" -"49136-5","CK_MBI","SERUM, PLASMA, or SR_PLS" -"2160-0","CREATININE","SERUM, PLASMA, or SR_PLS" -"38483-4","CREATININE","BLOOD" -"14682-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"14682-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"14682-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"14682-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"21232-4","CREATININE","BLOOD" -"21232-4","CREATININE","BLOOD" -"21232-4","CREATININE","BLOOD" -"21232-4","CREATININE","BLOOD" -"35203-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"35203-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"35203-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"35203-9","CREATININE","SERUM, PLASMA, or SR_PLS" -"44784-7","CREATININE","SERUM, PLASMA, or SR_PLS" -"44784-7","CREATININE","SERUM, PLASMA, or SR_PLS" -"44784-7","CREATININE","SERUM, PLASMA, or SR_PLS" -"44784-7","CREATININE","SERUM, PLASMA, or SR_PLS" -"59826-8","CREATININE","BLOOD" -"59826-8","CREATININE","BLOOD" -"59826-8","CREATININE","BLOOD" -"59826-8","CREATININE","BLOOD" -"14775-1","HGB","BLOOD" -"14775-1","HGB","BLOOD" -"14775-1","HGB","BLOOD" -"14775-1","HGB","BLOOD" -"20509-6","HGB","BLOOD" -"20509-6","HGB","BLOOD" -"20509-6","HGB","BLOOD" -"20509-6","HGB","BLOOD" -"30313-1","HGB","BLOOD" -"30313-1","HGB","BLOOD" -"30313-1","HGB","BLOOD" -"30313-1","HGB","BLOOD" -"30350-3","HGB","BLOOD" -"30350-3","HGB","BLOOD" -"30350-3","HGB","BLOOD" -"30350-3","HGB","BLOOD" -"30351-1","HGB","BLOOD" -"30351-1","HGB","BLOOD" -"30351-1","HGB","BLOOD" -"30351-1","HGB","BLOOD" -"30352-9","HGB","BLOOD" -"30352-9","HGB","BLOOD" -"30352-9","HGB","BLOOD" -"30352-9","HGB","BLOOD" -"55782-7","HGB","BLOOD" -"55782-7","HGB","BLOOD" -"55782-7","HGB","BLOOD" -"55782-7","HGB","BLOOD" -"59260-0","HGB","BLOOD" -"59260-0","HGB","BLOOD" -"59260-0","HGB","BLOOD" -"59260-0","HGB","BLOOD" -"718-7","HGB","BLOOD" -"34714-6","INR","BLOOD" -"34714-6","INR","BLOOD" -"34714-6","INR","BLOOD" -"34714-6","INR","BLOOD" -"46418-0","INR","BLOOD" -"46418-0","INR","BLOOD" -"46418-0","INR","BLOOD" -"46418-0","INR","BLOOD" -"6301-6","INR","PPP" -"6301-6","INR","PPP" -"6301-6","INR","PPP" -"6301-6","INR","PPP" -"10839-9","TROP_I","SERUM, PLASMA, or SR_PLS" -"10839-9","TROP_I","SERUM, PLASMA, or SR_PLS" -"10839-9","TROP_I","SERUM, PLASMA, or SR_PLS" -"10839-9","TROP_I","SERUM, PLASMA, or SR_PLS" -"16255-2","TROP_I","SERUM, PLASMA, or SR_PLS" -"16255-2","TROP_I","SERUM, PLASMA, or SR_PLS" -"16255-2","TROP_I","SERUM, PLASMA, or SR_PLS" -"16255-2","TROP_I","SERUM, PLASMA, or SR_PLS" -"42757-5","TROP_I","BLOOD" -"42757-5","TROP_I","BLOOD" -"42757-5","TROP_I","BLOOD" -"42757-5","TROP_I","BLOOD" -"49563-0","TROP_I","SERUM, PLASMA, or SR_PLS" -"49563-0","TROP_I","SERUM, PLASMA, or SR_PLS" -"49563-0","TROP_I","SERUM, PLASMA, or SR_PLS" -"49563-0","TROP_I","SERUM, PLASMA, or SR_PLS" -"33204-9","TROP_T_QL","SERUM, PLASMA, or SR_PLS" -"33204-9","TROP_T_QL","SERUM, PLASMA, or SR_PLS" -"33204-9","TROP_T_QL","SERUM, PLASMA, or SR_PLS" -"33204-9","TROP_T_QL","SERUM, PLASMA, or SR_PLS" -"48426-1","TROP_T_QL","BLOOD" -"48426-1","TROP_T_QL","BLOOD" -"48426-1","TROP_T_QL","BLOOD" -"48426-1","TROP_T_QL","BLOOD" -"48425-3","TROP_T_QN","BLOOD" -"48425-3","TROP_T_QN","BLOOD" -"48425-3","TROP_T_QN","BLOOD" -"48425-3","TROP_T_QN","BLOOD" -"6597-9","TROP_T_QN","BLOOD" -"6597-9","TROP_T_QN","BLOOD" -"6597-9","TROP_T_QN","BLOOD" -"6597-9","TROP_T_QN","BLOOD" -"6598-7","TROP_T_QN","SERUM, PLASMA, or SR_PLS" -"6598-7","TROP_T_QN","SERUM, PLASMA, or SR_PLS" -"6598-7","TROP_T_QN","SERUM, PLASMA, or SR_PLS" -"6598-7","TROP_T_QN","SERUM, PLASMA, or SR_PLS" diff --git a/Oracle/load_pcornet_mapping.sh b/Oracle/load_pcornet_mapping.sh deleted file mode 100644 index 651d6cc..0000000 --- a/Oracle/load_pcornet_mapping.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -set -e - -# Expected environment variables -# Database SID -# export sid= - -# User and password for CDM user -# export pcornet_cdm_user= -# export pcornet_cdm= - -# Create/Load the local path mapping and ontology update tables -sqlplus /nolog < 1; - -How many of each spec_order? - -select count(*), spec_order, dose_pref from medication_id_to_best_rxcui -group by spec_order, dose_pref order by 1 desc; - -27825 1) Semantic generic clinical drug (SCD) 1) Clarity -17527 1) Semantic generic clinical drug (SCD) 2) GCN - 3475 *null* lots of IVP. TODO: med mixes - 2888 9) HERON mapping misc. 2) GCN - 2550 9) HERON mapping misc. 1) Clarity - 421 1) Semantic generic clinical drug (SCD) 3) NDC - 295 3) Generic drug pack (GPCK) 1) Clarity - 177 3) Generic drug pack (GPCK) 2) GCN - 97 2) Semantic Branded clinical drug (SBC) 3) NDC - 4 9) HERON mapping misc. 3) NDC - 2 4) Branded drug pack (BPCK) 3) NDC - 1 3) Generic drug pack (GPCK) 3) NDC - -Be sure RXCUI for 0.4 ML ENOXAPARIN 100 MG/ML SC SYRG includes the 0.4 ML dose info: -select concept_cd, name_char, rxcui, rxnorm_str, spec_order, dose_pref from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:85052'; -- -> 854235, 1) SCD - -select concept_cd, name_char, rxcui, rxnorm_str, spec_order, dose_pref from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:85051'; -- -> 854248, 1) SCD - -This one goes from "LEVOTHYROXINE PO" to "Levothyroxine Sodium 0.1 MG Oral Tablet"; is that right? -select concept_cd, name_char, rxcui, rxnorm_str, spec_order, dose_pref from medication_id_to_best_rxcui where concept_cd = 'KUH|MEDICATION_ID:150171'; -- -> 892246, 1) SCD -*/ - - -insert into "&&i2b2_meta_schema".PCORNET_MED -with -terms_rx as ( - select - best.rxcui mapped_rxcui, ht.* - from - "&&i2b2_meta_schema"."&&terms_table" ht - left join medication_id_to_best_rxcui best on best.concept_cd = ht.c_basecode - where c_fullname like '\i2b2\Medications%' - and c_basecode not like 'NDC:%' -- We'll handle NDCs seperately below - and ht.c_visualattributes not like '%H%' - ) -select - rx.c_hlevel + 1 c_hlevel, - replace(rx.c_fullname, '\i2b2\Medications\', '\PCORI\MEDICATION\RXNORM_CUI\') c_fullname, - rx.c_name, rx.c_synonym_cd, rx.c_visualattributes, - rx.c_totalnum, rx.c_basecode, rx.c_metadataxml, rx.c_facttablecolumn, rx.c_tablename, - rx.c_columnname, rx.c_columndatatype, rx.c_operator, rx.c_dimcode, rx.c_comment, - rx.c_tooltip, rx.m_applied_path, rx.update_date, rx.download_date, rx.import_date, - rx.sourcesystem_cd, rx.valuetype_cd, rx.m_exclusion_cd, rx.c_path, rx.c_symbol, - rx.rxcui pcori_basecode, rx.rxcui pcori_cui, - -- Don't worry about NDCs for now - used for dispensing - null pcori_ndc from ( - select - trx.*, - case - when trx.mapped_rxcui is not null then trx.mapped_rxcui - when trx.c_basecode like 'RXCUI:%' then replace(trx.c_basecode, 'RXCUI:', '') - else null - end rxcui - from terms_rx trx - --order by trx.c_hlevel - ) rx; - --- Handle mapping NDC codes for DISPENSING -insert into "&&i2b2_meta_schema".PCORNET_MED -select - c_hlevel + 1 c_hlevel, - replace(c_fullname, '\i2b2\Medications\', '\PCORI\MEDICATION\RXNORM_CUI\') c_fullname, - c_name, c_synonym_cd, c_visualattributes, - c_totalnum, c_basecode, c_metadataxml, c_facttablecolumn, c_tablename, - c_columnname, c_columndatatype, c_operator, c_dimcode, c_comment, - c_tooltip, m_applied_path, update_date, download_date, import_date, - sourcesystem_cd, valuetype_cd, m_exclusion_cd, c_path, c_symbol, - null pcori_basecode, null pcori_cui, - substr(c_basecode, 5) pcori_ndc -from "&&i2b2_meta_schema"."&&terms_table" -where c_fullname like '\i2b2\Medications%' - and c_basecode like 'NDC:%' - and length(substr(c_basecode, 5)) < 12 -- Make sure we have 11 digit NDC -; - -delete -from "&&i2b2_meta_schema".PCORNET_MED where sourcesystem_cd='MAPPING'; - - -insert into "&&i2b2_meta_schema".PCORNET_MED -with med_mod_mapping as ( - select distinct pcori_path, i2b2.c_fullname, i2b2.c_basecode, i2b2.c_name - FROM "&&i2b2_meta_schema".pcornet_med - join pcornet_mapping on pcornet_mapping.PCORI_PATH = pcornet_med.c_fullname and pcornet_mapping.local_path is not null - join "&&i2b2_meta_schema"."&&terms_table" i2b2 on i2b2.c_fullname like pcornet_mapping.local_path || '%' - where i2b2.c_basecode is not null - ) -SELECT pcornet_med.C_HLEVEL+1, - pcornet_med.C_FULLNAME || med_mod_mapping.c_basecode || '\' as C_FULLNAME, - med_mod_mapping.c_basecode || ' ' || med_mod_mapping.c_name as C_NAME, - pcornet_med.C_SYNONYM_CD, - pcornet_med.C_VISUALATTRIBUTES, - pcornet_med.C_TOTALNUM, - med_mod_mapping.c_basecode as C_BASECODE, - pcornet_med.C_METADATAXML, - pcornet_med.C_FACTTABLECOLUMN, - pcornet_med.C_TABLENAME, - pcornet_med.C_COLUMNNAME, - pcornet_med.C_COLUMNDATATYPE, - pcornet_med.C_OPERATOR, - pcornet_med.C_FULLNAME || med_mod_mapping.c_basecode || '\' as C_DIMCODE, - pcornet_med.C_COMMENT, - pcornet_med.C_TOOLTIP, - pcornet_med.M_APPLIED_PATH, - pcornet_med.UPDATE_DATE, - pcornet_med.DOWNLOAD_DATE, - pcornet_med.IMPORT_DATE, - 'MAPPING' as SOURCESYSTEM_CD, - pcornet_med.VALUETYPE_CD, - pcornet_med.M_EXCLUSION_CD, - pcornet_med.C_PATH, - pcornet_med.C_SYMBOL, - pcornet_med.PCORI_BASECODE, - null pcori_cui, - null pcori_ndc -from med_mod_mapping -join "&&i2b2_meta_schema".pcornet_med on med_mod_mapping.PCORI_PATH = pcornet_med.c_fullname -; - -commit; - -/* Replace SCILHS PCORNet Labs hierarchy with our local LOINC hierarchy with - adjustment to make it "SCILHS like". -*/ - -/* Remove existing SCIHLS Labs hierarchy */ -truncate table "&&i2b2_meta_schema".pcornet_lab; - -/* Insert LOINC codes from local hierarchy, setting c_basecode to appropriate - LAB_NAME for common PCORNet labs. */ -insert into "&&i2b2_meta_schema".pcornet_lab -select distinct - lc.C_HLEVEL, - replace(lc.C_FULLNAME, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_fullname, - lc.C_NAME, - lc.C_SYNONYM_CD, - lc.C_VISUALATTRIBUTES, - lc.C_TOTALNUM, - case - when llm.lab_name is not null then 'LAB_NAME:'|| llm.lab_name - else lc.c_basecode - end c_basecode, - to_char(lc.C_METADATAXML), - lc.C_FACTTABLECOLUMN, - lc.C_TABLENAME, - lc.C_COLUMNNAME, - lc.C_COLUMNDATATYPE, - lc.C_OPERATOR, - lc.C_DIMCODE, - to_char(lc.C_COMMENT), - lc.C_TOOLTIP, - lc.M_APPLIED_PATH, - lc.UPDATE_DATE, - lc.DOWNLOAD_DATE, - lc.IMPORT_DATE, - lc.SOURCESYSTEM_CD, - lc.VALUETYPE_CD, - lc.M_EXCLUSION_CD, - lc.C_PATH, - lc.C_SYMBOL, - llm.pcori_specimen_source, - replace(lc.c_basecode, 'LOINC:', '') pcori_basecode -from "&&i2b2_meta_schema"."&&terms_table" lc -left outer join lab_loinc_mapping llm - on lc.c_basecode = ('LOINC:' || llm.loinc_code) -where lc.c_basecode like 'LOINC:%' -; - -/* Insert child KUH|COMPONENT_ID nodes, setting pcori_basecode, and c_path to - that of it's parent. */ -insert into "&&i2b2_meta_schema".pcornet_lab -with parent_loinc_codes as ( -- LOINC codes with children LOINC codes - select p_loinc.* - from "&&i2b2_meta_schema"."&&terms_table" p_loinc - join "&&i2b2_meta_schema"."&&terms_table" c_loinc - on c_loinc.c_fullname like (p_loinc.c_fullname || '%') - and p_loinc.c_basecode like 'LOINC:%' - and c_loinc.c_basecode like 'LOINC:%' - and p_loinc.c_basecode!=c_loinc.c_basecode -) -, children_loinc_codes as ( -- LOINC codes without children LOINC codes - select clc.* - from "&&i2b2_meta_schema"."&&terms_table" clc - where clc.c_basecode like 'LOINC:%' - and clc.c_basecode not in ( - select c_basecode from parent_loinc_codes - ) -) -select distinct - ccc.C_HLEVEL, - replace(ccc.C_FULLNAME, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_fullname, - ccc.C_NAME, - ccc.C_SYNONYM_CD, - ccc.C_VISUALATTRIBUTES, - ccc.C_TOTALNUM, - ccc.C_BASECODE, - to_char(ccc.C_METADATAXML), - ccc.C_FACTTABLECOLUMN, - ccc.C_TABLENAME, - ccc.C_COLUMNNAME, - ccc.C_COLUMNDATATYPE, - ccc.C_OPERATOR, - ccc.C_DIMCODE, - to_char(ccc.C_COMMENT), - ccc.C_TOOLTIP, - ccc.M_APPLIED_PATH, - ccc.UPDATE_DATE, - ccc.DOWNLOAD_DATE, - ccc.IMPORT_DATE, - ccc.SOURCESYSTEM_CD, - ccc.VALUETYPE_CD, - ccc.M_EXCLUSION_CD, - replace(clc.C_FULLNAME, '\i2b2\Laboratory Tests\', '\PCORI\LAB_RESULT_CM\') c_path, - ccc.C_SYMBOL, - llm.pcori_specimen_source, - replace(clc.c_basecode, 'LOINC:', '') pcori_basecode -from children_loinc_codes clc -join "&&i2b2_meta_schema"."&&terms_table" ccc - on ccc.c_fullname like (clc.c_fullname || '%') - and ccc.c_basecode like 'KUH|COMPONENT_ID:%' -- TODO: Generalize for other sites. -left outer join lab_loinc_mapping llm - on clc.c_basecode = ('LOINC:' || llm.loinc_code) -; - -commit; - diff --git a/Oracle/pcornet_mapping_ddl.sql b/Oracle/pcornet_mapping_ddl.sql deleted file mode 100644 index 12d6e87..0000000 --- a/Oracle/pcornet_mapping_ddl.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE TABLE pcornet_mapping ( - "PCORI_PATH" VARCHAR2(2000), - "LOCAL_PATH" VARCHAR2(2000) - ); diff --git a/Oracle/pcornet_ontology_updates.csv b/Oracle/pcornet_ontology_updates.csv deleted file mode 100644 index b5c10e2..0000000 --- a/Oracle/pcornet_ontology_updates.csv +++ /dev/null @@ -1,9 +0,0 @@ -"C_HLEVEL","C_FULLNAME","C_NAME","C_SYNONYM_CD","C_VISUALATTRIBUTES","C_TOTALNUM","C_BASECODE","C_METADATAXML","C_FACTTABLECOLUMN","C_TABLENAME","C_COLUMNNAME","C_COLUMNDATATYPE","C_OPERATOR","C_DIMCODE","C_COMMENT","C_TOOLTIP","M_APPLIED_PATH","UPDATE_DATE","DOWNLOAD_DATE","IMPORT_DATE","SOURCESYSTEM_CD","VALUETYPE_CD","M_EXCLUSION_CD","C_PATH","C_SYMBOL","PCORI_BASECODE" -2,"\PCORI\DEMOGRAPHIC\HISPANIC\","Ethnicity","N","CAE",,,"3.22014-05-09T11:10:26.266-04:00EnumNoUnknownYes ","concept_cd","CONCEPT_DIMENSION","concept_path","T","like","\PCORI\DEMOGRAPHIC\HISPANIC\",,"Ethnicity","@","2014-05-09 11:10:28","2014-05-09 11:10:28","2014-05-09 11:10:28","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\","HISPANIC", -3,"\PCORI\DEMOGRAPHIC\HISPANIC\NI\","No information","N","LAE",,"ETHNICITY:NI",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","N","IS","NULL",,"No information: A data field is present in the source system, but the source value is null or blank.","@","2014-06-02 19:40:30","2014-06-02 19:40:30","2014-06-02 19:40:30","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","NI","NI" -3,"\PCORI\DEMOGRAPHIC\HISPANIC\N\","Non-Hispanic","N","LIE",,"ETHNICITY:NOTHISPANIC",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'N'",,"Non-Hispanic","@","2014-05-09 11:12:04","2014-05-09 11:12:04","2014-05-09 11:12:04","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","N","N" -3,"\PCORI\DEMOGRAPHIC\HISPANIC\OT\","Other","N","LAE",,"ETHNICITY:OT",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'OT'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2014-06-02 19:44:16","2014-06-02 19:44:16","2014-06-02 19:44:16","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","OT","OT" -3,"\PCORI\DEMOGRAPHIC\HISPANIC\UN\","Unknown","N","LAE",,"ETHNICITY:UN",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'UN','u','unk'",,"Unknown: A data field is present in the source system, but the source value explicitly denotes an unknown value.","@","2014-06-02 19:43:08","2014-06-02 19:43:08","2014-06-02 19:43:08","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","UN","UN" -3,"\PCORI\DEMOGRAPHIC\HISPANIC\Y\","Hispanic","N","LAE",,"ETHNICITY:HISPANIC",,"PATIENT_NUM","PATIENT_DIMENSION","ETHNICITY_CD","T","IN","'Y','hib','his/black','his/white','hiw','hispanic'",,"A person of Cuban, Mexican, Puerto Rican, South or Central American, or other Spanish culture or origin, regardless of race. \ Hisptanic","@","2014-05-09 15:21:33","2014-05-09 11:11:12","2014-05-09 11:11:12","PCORNET_CDM",,,"\PCORI\DEMOGRAPHIC\HISPANIC\","Y","Y" -3,"\PCORI\DEMOGRAPHIC\RACE\NI\","No information","N","LAE",,"RACE:NI",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","N","IS","'@'",,"No information: A data field is present in the source system, but the source value is null or blank.","@","2016-03-30 16:48:29","2016-03-30 16:48:29","2016-03-30 16:48:29","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","NI","NI" -3,"\PCORI\DEMOGRAPHIC\RACE\OT\","Other","N","LAE",,"RACE:OT",,"PATIENT_NUM","PATIENT_DIMENSION","RACE_CD","T","IN","'OT','o','d','deferred'",,"Other: A data field is present in the source system, but the source value cannot be mapped to the CDM.","@","2016-03-30 16:48:29","2016-03-30 16:48:29","2016-03-30 16:48:29","PCORNET_CDM",,"","\PCORI\DEMOGRAPHIC\RACE\","OT","OT" diff --git a/Oracle/pcornet_ontology_updates.ctl b/Oracle/pcornet_ontology_updates.ctl deleted file mode 100644 index c0d966d..0000000 --- a/Oracle/pcornet_ontology_updates.ctl +++ /dev/null @@ -1,32 +0,0 @@ -options (direct=true, errors=0, skip=1) -load data -truncate into table PCORNET_ONTOLOGY_UPDATES -fields terminated by ',' optionally enclosed by '"' -trailing nullcols( - C_HLEVEL, - C_FULLNAME, - C_NAME CHAR(2000), - C_SYNONYM_CD, - C_VISUALATTRIBUTES, - C_TOTALNUM, - C_BASECODE, - C_METADATAXML CHAR(100000), - C_FACTTABLECOLUMN, - C_TABLENAME, - C_COLUMNNAME, - C_COLUMNDATATYPE, - C_OPERATOR, - C_DIMCODE CHAR(700), - C_COMMENT, - C_TOOLTIP CHAR(900), - M_APPLIED_PATH, - UPDATE_DATE DATE 'YYYY-MM-DD HH24:MI:SS', - DOWNLOAD_DATE DATE 'YYYY-MM-DD HH24:MI:SS', - IMPORT_DATE DATE 'YYYY-MM-DD HH24:MI:SS', - SOURCESYSTEM_CD, - VALUETYPE_CD, - M_EXCLUSION_CD, - C_PATH, - C_SYMBOL, - PCORI_BASECODE - ) \ No newline at end of file diff --git a/Oracle/pcornet_ontology_updates_ddl.sql b/Oracle/pcornet_ontology_updates_ddl.sql deleted file mode 100644 index 82ec392..0000000 --- a/Oracle/pcornet_ontology_updates_ddl.sql +++ /dev/null @@ -1,29 +0,0 @@ -CREATE TABLE pcornet_ontology_updates ( - "C_HLEVEL" NUMBER(22,0) NOT NULL, - "C_FULLNAME" VARCHAR2(700) NOT NULL, - "C_NAME" VARCHAR2(2000) NOT NULL, - "C_SYNONYM_CD" CHAR(1) NOT NULL, - "C_VISUALATTRIBUTES" CHAR(3) NOT NULL, - "C_TOTALNUM" NUMBER(22,0) NULL, - "C_BASECODE" VARCHAR2(50) NULL, - "C_METADATAXML" CLOB NULL, - "C_FACTTABLECOLUMN" VARCHAR2(50) NOT NULL, - "C_TABLENAME" VARCHAR2(50) NOT NULL, - "C_COLUMNNAME" VARCHAR2(50) NOT NULL, - "C_COLUMNDATATYPE" VARCHAR2(50) NOT NULL, - "C_OPERATOR" VARCHAR2(10) NOT NULL, - "C_DIMCODE" VARCHAR2(700) NOT NULL, - "C_COMMENT" CLOB NULL, - "C_TOOLTIP" VARCHAR2(900) NULL, - "M_APPLIED_PATH" VARCHAR2(700) NOT NULL, - "UPDATE_DATE" DATE NOT NULL, - "DOWNLOAD_DATE" DATE NULL, - "IMPORT_DATE" DATE NULL, - "SOURCESYSTEM_CD" VARCHAR2(50) NULL, - "VALUETYPE_CD" VARCHAR2(50) NULL, - "M_EXCLUSION_CD" VARCHAR2(25) NULL, - "C_PATH" VARCHAR2(700) NULL, - "C_SYMBOL" VARCHAR2(50) NULL, - "PCORI_BASECODE" VARCHAR2(50) NULL -) -; diff --git a/Oracle/run-i2p-transform.sh b/Oracle/run-i2p-transform.sh index f0a4b7b..ad1544e 100644 --- a/Oracle/run-i2p-transform.sh +++ b/Oracle/run-i2p-transform.sh @@ -1,32 +1,9 @@ #!/bin/bash set -e -# Expected environment variables (put there by Jenkins, etc.) - -# Database SID -#export sid= -# User and password for CDM user -#export pcornet_cdm_user= -#export pcornet_cdm= - -#export i2b2_data_schema= -#export i2b2_meta_schema= -#export datamart_id= -#export datamart_name= -#export network_id= -#export network_name= -# export i2b2_etl_schema= -# export min_pat_list_date_dd_mon_rrrr= -# export min_visit_date_dd_mon_rrrr= -# export enrollment_months_back=36 - -# All i2b2 terms - used for local path mapping -#export terms_table= - python load_csv.py harvest_local harvest_local.csv harvest_local.ctl pcornet_cdm_user pcornet_cdm python load_csv.py PMN_LabNormal pmn_labnormal.csv pmn_labnormal.ctl pcornet_cdm_user pcornet_cdm python load_csv.py lab_loinc_mapping lab_loinc_mapping.csv lab_loinc_mapping.ctl pcornet_cdm_user pcornet_cdm -. ./load_pcornet_mapping.sh # Run some tests sqlplus /nolog < 0 then 1 else 1/0 end obs_fact_meds_populated from ( select count(*) qty from observation_fact_meds ); - EOF -# Insert local terms as leaves of the PCORNet terms and run the transform +########### Initialize Schema ########### +if [ ${initialize_schema} == 'true' ] +then + sqlplus /nolog <