From 44e72c97bab27332681dd36c229c5aef64ffb7a3 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Tue, 19 Jan 2021 16:24:23 +0200 Subject: [PATCH 1/6] Update load_stage.sql 1. manual concepts ( 'U07.1', 'U07.0') were deleted because they had appeared in the source 2. LOINC was added twice as the filter at step #10 (Update domain_id for ICD10CM using mapping) --- ICD10CM/load_stage.sql | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/ICD10CM/load_stage.sql b/ICD10CM/load_stage.sql index bd5662742..e23cb5229 100644 --- a/ICD10CM/load_stage.sql +++ b/ICD10CM/load_stage.sql @@ -70,13 +70,7 @@ SELECT SUBSTR(CASE ) AS valid_start_date, TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, NULL AS invalid_reason -FROM sources.icd10cm - ---manual concepts ---https://www.cdc.gov/nchs/data/icd/Vaping-Announcement-final-12-09-19.pdf -UNION ALL -VALUES ('Emergency use of U07.1 | COVID-19','Condition','ICD10CM','4-char billing code', NULL,'U07.1',TO_DATE('20200401','yyyymmdd'),TO_DATE('20991231','yyyymmdd'),NULL), - ('Emergency use of U07.0 | Vaping-related disorder','Condition','ICD10CM','4-char billing code',NULL,'U07.0',TO_DATE('20200401','yyyymmdd'),TO_DATE('20991231','yyyymmdd'),NULL); +FROM sources.icd10cm; --4. Add ICD10CM to SNOMED manual mappings DO $_$ @@ -146,7 +140,7 @@ WHERE c2.concept_code LIKE c1.concept_code || '%' ); DROP INDEX trgm_idx; ---10. Update domain_id for ICD10CM from SNOMED +--10. Update domain_id for ICD10CM from SNOMED and LOINC UPDATE concept_stage cs SET domain_id = i.domain_id FROM ( @@ -172,7 +166,7 @@ FROM ( AND cs1.vocabulary_id = 'ICD10CM' JOIN concept c2 ON c2.concept_code = crs.concept_code_2 AND c2.vocabulary_id = crs.vocabulary_id_2 - AND c2.vocabulary_id = 'SNOMED' + AND c2.vocabulary_id in ( 'SNOMED', 'LOINC') WHERE crs.relationship_id = 'Maps to' AND crs.invalid_reason IS NULL @@ -198,7 +192,7 @@ FROM ( JOIN concept c1 ON c1.concept_id = cr.concept_id_1 AND c1.vocabulary_id = 'ICD10CM' JOIN concept c2 ON c2.concept_id = cr.concept_id_2 - AND c2.vocabulary_id = 'SNOMED' + AND c2.vocabulary_id in ( 'SNOMED', 'LOINC') JOIN concept_stage cs1 ON cs1.concept_code = c1.concept_code AND cs1.vocabulary_id = c1.vocabulary_id WHERE cr.relationship_id = 'Maps to' From 07b43460c4eedaee04d882f55729dfbe4aad97ff Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Tue, 19 Jan 2021 17:28:41 +0200 Subject: [PATCH 2/6] Update load_stage.sql 1. concept_manual and concept_synonym_manual were included (steps 4 and 5) --- ICD10CM/load_stage.sql | 43 ++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/ICD10CM/load_stage.sql b/ICD10CM/load_stage.sql index e23cb5229..98abeee30 100644 --- a/ICD10CM/load_stage.sql +++ b/ICD10CM/load_stage.sql @@ -72,37 +72,49 @@ SELECT SUBSTR(CASE NULL AS invalid_reason FROM sources.icd10cm; ---4. Add ICD10CM to SNOMED manual mappings +--4. Add manual concepts or changes +DO $_$ +BEGIN + PERFORM VOCABULARY_PACK.ProcessManualConcepts(); +END $_$; + +--5. Working with manual synonyms +DO $_$ +BEGIN + PERFORM VOCABULARY_PACK.ProcessManualSynonyms(); +END $_$; + +--6. Add ICD10CM to SNOMED manual mappings DO $_$ BEGIN PERFORM VOCABULARY_PACK.ProcessManualRelationships(); END $_$; ---5. Working with replacement mappings +--7. Working with replacement mappings DO $_$ BEGIN PERFORM VOCABULARY_PACK.CheckReplacementMappings(); END $_$; ---6. Add mapping from deprecated to fresh concepts +--8. Add mapping from deprecated to fresh concepts DO $_$ BEGIN PERFORM VOCABULARY_PACK.AddFreshMAPSTO(); END $_$; ---7. Deprecate 'Maps to' mappings to deprecated and upgraded concepts +--9. Deprecate 'Maps to' mappings to deprecated and upgraded concepts DO $_$ BEGIN PERFORM VOCABULARY_PACK.DeprecateWrongMAPSTO(); END $_$; ---8. Delete ambiguous 'Maps to' mappings +--10. Delete ambiguous 'Maps to' mappings DO $_$ BEGIN PERFORM VOCABULARY_PACK.DeleteAmbiguousMAPSTO(); END $_$; ---9. Add "subsumes" relationship between concepts where the concept_code is like of another +--11. Add "subsumes" relationship between concepts where the concept_code is like of another CREATE INDEX IF NOT EXISTS trgm_idx ON concept_stage USING GIN (concept_code devv5.gin_trgm_ops); --for LIKE patterns ANALYZE concept_stage; INSERT INTO concept_relationship_stage ( @@ -140,7 +152,7 @@ WHERE c2.concept_code LIKE c1.concept_code || '%' ); DROP INDEX trgm_idx; ---10. Update domain_id for ICD10CM from SNOMED and LOINC +--12. Update domain_id for ICD10CM from SNOMED and LOINC UPDATE concept_stage cs SET domain_id = i.domain_id FROM ( @@ -208,11 +220,11 @@ FROM ( WHERE i.concept_code = cs.concept_code AND cs.vocabulary_id = 'ICD10CM'; ---11. Check for NULL in domain_id +--13. Check for NULL in domain_id ALTER TABLE concept_stage ALTER COLUMN domain_id SET NOT NULL; ALTER TABLE concept_stage ALTER COLUMN domain_id DROP NOT NULL; ---12. Load into concept_synonym_stage name from ICD10CM +--14. Load into concept_synonym_stage name from ICD10CM INSERT INTO concept_synonym_stage ( synonym_concept_code, synonym_name, @@ -233,20 +245,15 @@ FROM ( SELECT short_name AS synonym_name, REGEXP_REPLACE(code, '([[:print:]]{3})([[:print:]]+)', '\1.\2') AS code FROM sources.icd10cm - ) AS s0 - -UNION ALL - -VALUES ('U07.1','Disease caused by Severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2)','ICD10CM',4180186), - ('U07.0','Vaping-related disorder','ICD10CM',4180186); + ) AS s0; ---13. Add mapping from deprecated to fresh concepts for 'Maps to value' +--15. Add mapping from deprecated to fresh concepts for 'Maps to value' DO $_$ BEGIN PERFORM VOCABULARY_PACK.AddFreshMapsToValue(); END $_$; ---14. Build reverse relationship. This is necessary for next point +--16. Build reverse relationship. This is necessary for next point INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -278,7 +285,7 @@ WHERE NOT EXISTS ( AND r.reverse_relationship_id = i.relationship_id ); ---15. Deprecate all relationships in concept_relationship that aren't exist in concept_relationship_stage +--17. Deprecate all relationships in concept_relationship that aren't exist in concept_relationship_stage INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, From 53a8e8627d696ab819cb3fc1ed43b9c672c9c8d3 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Tue, 19 Jan 2021 18:23:32 +0200 Subject: [PATCH 3/6] Create readme.md 1. readme for manual work was created --- ICD10CM/manual_work/readme.md | 46 +++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 ICD10CM/manual_work/readme.md diff --git a/ICD10CM/manual_work/readme.md b/ICD10CM/manual_work/readme.md new file mode 100644 index 000000000..86377e166 --- /dev/null +++ b/ICD10CM/manual_work/readme.md @@ -0,0 +1,46 @@ +### Manual content processing: +1.Extract the following csv file into the concept_manual table: https://drive.google.com/file/d/1iCmdHud7Y296SpeOan0vNNJEsg4qBsin/view?usp=sharing +File is generated using the query: + +`SELECT concept_name, + domain_id, + vocabulary_id, + concept_class_id, + standard_concept, + concept_code, + valid_start_date, + valid_end_date, + invalid_reason +FROM concept_manual +ORDER BY vocabulary_id, concept_code, invalid_reason, valid_start_date, valid_end_date, concept_name;` + +2.Extract the following csv file into the concept_synonym_manual table: https://drive.google.com/file/d/1C9qVJwR369y9Jk02iS-qK45Gn5iGGRJr/view?usp=sharing + +`SELECT synonym_name, + synonym_concept_code, + synonym_vocabulary_id, + language_concept_id +FROM concept_synonym_manual +ORDER BY synonym_vocabulary_id, synonym_concept_code, language_concept_id, synonym_name;` + +3.Extract the following csv file into the concept_relationship_manual table: https://drive.google.com/file/d/1uUrkcknqIogs4Os0Hqu51WjSPo0C5mvH/view?usp=sharing + +`SELECT concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason +FROM concept_relationship_manual +ORDER BY vocabulary_id_1, vocabulary_id_2, relationship_id, concept_code_1, concept_code_2, invalid_reason, valid_start_date, valid_end_date;` + +##### csv format: +- delimiter: ',' +- encoding: 'UTF8' +- header: ON +- decimal symbol: '.' +- quote escape: with backslash \ +- quote always: FALSE +- NULL string: empty From 79eb92084f83286d8fceb656a034be8ef640d89c Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Tue, 19 Jan 2021 18:27:23 +0200 Subject: [PATCH 4/6] Update readme.md --- ICD10CM/manual_work/readme.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/ICD10CM/manual_work/readme.md b/ICD10CM/manual_work/readme.md index 86377e166..741efeec0 100644 --- a/ICD10CM/manual_work/readme.md +++ b/ICD10CM/manual_work/readme.md @@ -1,8 +1,8 @@ ### Manual content processing: 1.Extract the following csv file into the concept_manual table: https://drive.google.com/file/d/1iCmdHud7Y296SpeOan0vNNJEsg4qBsin/view?usp=sharing File is generated using the query: - -`SELECT concept_name, +```sql +SELECT concept_name, domain_id, vocabulary_id, concept_class_id, @@ -12,20 +12,21 @@ File is generated using the query: valid_end_date, invalid_reason FROM concept_manual -ORDER BY vocabulary_id, concept_code, invalid_reason, valid_start_date, valid_end_date, concept_name;` - +ORDER BY vocabulary_id, concept_code, invalid_reason, valid_start_date, valid_end_date, concept_name; +``` 2.Extract the following csv file into the concept_synonym_manual table: https://drive.google.com/file/d/1C9qVJwR369y9Jk02iS-qK45Gn5iGGRJr/view?usp=sharing - -`SELECT synonym_name, +```sql +SELECT synonym_name, synonym_concept_code, synonym_vocabulary_id, language_concept_id FROM concept_synonym_manual ORDER BY synonym_vocabulary_id, synonym_concept_code, language_concept_id, synonym_name;` - +``` 3.Extract the following csv file into the concept_relationship_manual table: https://drive.google.com/file/d/1uUrkcknqIogs4Os0Hqu51WjSPo0C5mvH/view?usp=sharing - -`SELECT concept_code_1, +File is generated using the query: +```sql +SELECT concept_code_1, concept_code_2, vocabulary_id_1, vocabulary_id_2, @@ -34,8 +35,8 @@ ORDER BY synonym_vocabulary_id, synonym_concept_code, language_concept_id, synon valid_end_date, invalid_reason FROM concept_relationship_manual -ORDER BY vocabulary_id_1, vocabulary_id_2, relationship_id, concept_code_1, concept_code_2, invalid_reason, valid_start_date, valid_end_date;` - +ORDER BY vocabulary_id_1, vocabulary_id_2, relationship_id, concept_code_1, concept_code_2, invalid_reason, valid_start_date, valid_end_date; +``` ##### csv format: - delimiter: ',' - encoding: 'UTF8' From ed8aacdda79227436697c5e345f619ff7a24c0a9 Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Tue, 19 Jan 2021 18:33:25 +0200 Subject: [PATCH 5/6] Update readme.md --- ICD10CM/manual_work/readme.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ICD10CM/manual_work/readme.md b/ICD10CM/manual_work/readme.md index 741efeec0..c91bb29f3 100644 --- a/ICD10CM/manual_work/readme.md +++ b/ICD10CM/manual_work/readme.md @@ -1,6 +1,5 @@ ### Manual content processing: -1.Extract the following csv file into the concept_manual table: https://drive.google.com/file/d/1iCmdHud7Y296SpeOan0vNNJEsg4qBsin/view?usp=sharing -File is generated using the query: +1.Extract the following csv file into the concept_manual table: https://drive.google.com/file/d/1iCmdHud7Y296SpeOan0vNNJEsg4qBsin/view?usp=sharing. The file was generated using the query: ```sql SELECT concept_name, domain_id, @@ -14,7 +13,7 @@ SELECT concept_name, FROM concept_manual ORDER BY vocabulary_id, concept_code, invalid_reason, valid_start_date, valid_end_date, concept_name; ``` -2.Extract the following csv file into the concept_synonym_manual table: https://drive.google.com/file/d/1C9qVJwR369y9Jk02iS-qK45Gn5iGGRJr/view?usp=sharing +2.Extract the following csv file into the concept_synonym_manual table: https://drive.google.com/file/d/1C9qVJwR369y9Jk02iS-qK45Gn5iGGRJr/view?usp=sharing. The file was generated using the query: ```sql SELECT synonym_name, synonym_concept_code, @@ -23,8 +22,7 @@ SELECT synonym_name, FROM concept_synonym_manual ORDER BY synonym_vocabulary_id, synonym_concept_code, language_concept_id, synonym_name;` ``` -3.Extract the following csv file into the concept_relationship_manual table: https://drive.google.com/file/d/1uUrkcknqIogs4Os0Hqu51WjSPo0C5mvH/view?usp=sharing -File is generated using the query: +3.Extract the following csv file into the concept_relationship_manual table: https://drive.google.com/file/d/1uUrkcknqIogs4Os0Hqu51WjSPo0C5mvH/view?usp=sharing. The file was generated using the query: ```sql SELECT concept_code_1, concept_code_2, From b557ba863fcc1f965dbbdba39275068a0aba347c Mon Sep 17 00:00:00 2001 From: Polina Talapova <42737398+p-talapova@users.noreply.github.com> Date: Tue, 19 Jan 2021 18:46:00 +0200 Subject: [PATCH 6/6] Update readme.md information about manual work was added --- ICD10CM/readme.md | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/ICD10CM/readme.md b/ICD10CM/readme.md index 0c6d4394b..fcde29a9b 100644 --- a/ICD10CM/readme.md +++ b/ICD10CM/readme.md @@ -1,16 +1,19 @@ -Update of ICD10CM +### ICD10CM upload / update ### -Prerequisites: -- Schema DevV5 with copies of tables concept, concept_relationship and concept_synonym from ProdV5, fully indexed. -- Schema UMLS -- SNOMED must be loaded first -- Working directory ICD10CM. +#### Prerequisites: #### +* Schema DevV5 with copies of tables concept, concept_relationship and concept_synonym from ProdV5, fully indexed. +* Schema UMLS +* SNOMED must be loaded first +* Working directory, e.g. dev_icd10cm -1. Run create_source_tables.sql -2. Download ICD10CM_FYYYYY_code_descriptions.zip or icd10cm_order_YYYY.txt from ftp://ftp.cdc.gov/pub/Health_Statistics/NCHS/Publications/ICD10CM/YYYY -3. Extract icd10cm_order_YYYY.txt (if needed) and rename to icd10cm.txt -4. Run in devv5 (with fresh vocabulary date and version, e.g. according "The 2018 ICD-10-CM codes are to be used from October 1, 2017 through September 30, 2018" so 20171001): +#### Sequence of actions #### +* Run create_source_tables.sql +* Download ICD10CM_FYYYYY_code_descriptions.zip or icd10cm_order_YYYY.txt from ftp://ftp.cdc.gov/pub/Health_Statistics/NCHS/Publications/ICD10CM/YYYY +* Extract icd10cm_order_YYYY.txt (if needed) and rename to icd10cm.txt +* Perform actions described in the folder of 'manual_work' +* Run in devv5 (with fresh vocabulary date and version, e.g. according "The 2018 ICD-10-CM codes are to be used from October 1, 2017 through September 30, 2018" so 20171001): +```sql SELECT sources.load_input_tables('ICD10CM',TO_DATE('20170428','YYYYMMDD'),'ICD10CM FY2017 code descriptions'); -5. Run load_stage.sql -Note: Load_stage generates list of the relationships that need to be checked and modified by the medical coder and after uploading this data to server load_stage continues making proper relationships using this manually created table -6. Run generic_update: devv5.GenericUpdate(); \ No newline at end of file +``` +* Run load_stage.sql. Note: Load_stage generates list of the relationships that need to be checked and modified by the medical coder and after uploading this data to server load_stage continues making proper relationships using this manually created table +* Run generic_update: devv5.GenericUpdate();