Merge pull request #442 from OHDSI/icd10cm-refresh-190121

Update load_stage.sql

ICD10CM/load_stage.sql

@@ -70,45 +70,51 @@ SELECT SUBSTR(CASE
 		) AS valid_start_date,
 	TO_DATE('20991231', 'yyyymmdd') AS valid_end_date,
 	NULL AS invalid_reason
-FROM sources.icd10cm
+FROM sources.icd10cm;
 
---manual concepts
---https://www.cdc.gov/nchs/data/icd/Vaping-Announcement-final-12-09-19.pdf
-UNION ALL
-VALUES ('Emergency use of U07.1 | COVID-19','Condition','ICD10CM','4-char billing code', NULL,'U07.1',TO_DATE('20200401','yyyymmdd'),TO_DATE('20991231','yyyymmdd'),NULL),
-	('Emergency use of U07.0 | Vaping-related disorder','Condition','ICD10CM','4-char billing code',NULL,'U07.0',TO_DATE('20200401','yyyymmdd'),TO_DATE('20991231','yyyymmdd'),NULL);
+--4. Add manual concepts or changes
+DO $_$
+BEGIN
+	PERFORM VOCABULARY_PACK.ProcessManualConcepts();
+END $_$;
 
---4. Add ICD10CM to SNOMED manual mappings
+--5. Working with manual synonyms
+DO $_$
+BEGIN
+	PERFORM VOCABULARY_PACK.ProcessManualSynonyms();
+END $_$;
+
+--6. Add ICD10CM to SNOMED manual mappings
 DO $_$
 BEGIN
 	PERFORM VOCABULARY_PACK.ProcessManualRelationships();
 END $_$;
 
---5. Working with replacement mappings
+--7. Working with replacement mappings
 DO $_$
 BEGIN
 	PERFORM VOCABULARY_PACK.CheckReplacementMappings();
 END $_$;
 
---6. Add mapping from deprecated to fresh concepts
+--8. Add mapping from deprecated to fresh concepts
 DO $_$
 BEGIN
 	PERFORM VOCABULARY_PACK.AddFreshMAPSTO();
 END $_$;
 
---7. Deprecate 'Maps to' mappings to deprecated and upgraded concepts
+--9. Deprecate 'Maps to' mappings to deprecated and upgraded concepts
 DO $_$
 BEGIN
 	PERFORM VOCABULARY_PACK.DeprecateWrongMAPSTO();
 END $_$;
 
---8. Delete ambiguous 'Maps to' mappings
+--10. Delete ambiguous 'Maps to' mappings
 DO $_$
 BEGIN
 	PERFORM VOCABULARY_PACK.DeleteAmbiguousMAPSTO();
 END $_$;
 
---9. Add "subsumes" relationship between concepts where the concept_code is like of another
+--11. Add "subsumes" relationship between concepts where the concept_code is like of another
 CREATE INDEX IF NOT EXISTS trgm_idx ON concept_stage USING GIN (concept_code devv5.gin_trgm_ops); --for LIKE patterns
 ANALYZE concept_stage;
 INSERT INTO concept_relationship_stage (
@@ -146,7 +152,7 @@ WHERE c2.concept_code LIKE c1.concept_code || '%'
 		);
 DROP INDEX trgm_idx;
 
---10. Update domain_id for ICD10CM from SNOMED
+--12. Update domain_id for ICD10CM from SNOMED and LOINC
 UPDATE concept_stage cs
 SET domain_id = i.domain_id
 FROM (
@@ -172,7 +178,7 @@ FROM (
 		AND cs1.vocabulary_id = 'ICD10CM'
 	JOIN concept c2 ON c2.concept_code = crs.concept_code_2
 		AND c2.vocabulary_id = crs.vocabulary_id_2
-		AND c2.vocabulary_id = 'SNOMED'
+		AND c2.vocabulary_id in ( 'SNOMED', 'LOINC')
 	WHERE crs.relationship_id = 'Maps to'
 		AND crs.invalid_reason IS NULL
 
@@ -198,7 +204,7 @@ FROM (
 	JOIN concept c1 ON c1.concept_id = cr.concept_id_1
 		AND c1.vocabulary_id = 'ICD10CM'
 	JOIN concept c2 ON c2.concept_id = cr.concept_id_2
-		AND c2.vocabulary_id = 'SNOMED'
+		AND c2.vocabulary_id in ( 'SNOMED', 'LOINC')
 	JOIN concept_stage cs1 ON cs1.concept_code = c1.concept_code
 		AND cs1.vocabulary_id = c1.vocabulary_id
 	WHERE cr.relationship_id = 'Maps to'
@@ -214,11 +220,11 @@ FROM (
 WHERE i.concept_code = cs.concept_code
 	AND cs.vocabulary_id = 'ICD10CM';
 
---11. Check for NULL in domain_id
+--13. Check for NULL in domain_id
 ALTER TABLE concept_stage ALTER COLUMN domain_id SET NOT NULL;
 ALTER TABLE concept_stage ALTER COLUMN domain_id DROP NOT NULL;
 
---12. Load into concept_synonym_stage name from ICD10CM
+--14. Load into concept_synonym_stage name from ICD10CM
 INSERT INTO concept_synonym_stage (
 	synonym_concept_code,
 	synonym_name,
@@ -239,20 +245,15 @@ FROM (
 	SELECT short_name AS synonym_name,
 		REGEXP_REPLACE(code, '([[:print:]]{3})([[:print:]]+)', '\1.\2') AS code
 	FROM sources.icd10cm
-	) AS s0
-
-UNION ALL
-
-VALUES ('U07.1','Disease caused by Severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2)','ICD10CM',4180186),
-	('U07.0','Vaping-related disorder','ICD10CM',4180186);
+	) AS s0;
 
---13. Add mapping from deprecated to fresh concepts for 'Maps to value'
+--15. Add mapping from deprecated to fresh concepts for 'Maps to value'
 DO $_$
 BEGIN
 	PERFORM VOCABULARY_PACK.AddFreshMapsToValue();
 END $_$;
 
---14. Build reverse relationship. This is necessary for next point
+--16. Build reverse relationship. This is necessary for next point
 INSERT INTO concept_relationship_stage (
 	concept_code_1,
 	concept_code_2,
@@ -284,7 +285,7 @@ WHERE NOT EXISTS (
 			AND r.reverse_relationship_id = i.relationship_id
 		);
 
---15. Deprecate all relationships in concept_relationship that aren't exist in concept_relationship_stage
+--17. Deprecate all relationships in concept_relationship that aren't exist in concept_relationship_stage
 INSERT INTO concept_relationship_stage (
 	concept_code_1,
 	concept_code_2,

ICD10CM/manual_work/readme.md

@@ -0,0 +1,45 @@
+### Manual content processing:
+1.Extract the following csv file into the concept_manual table: https://drive.google.com/file/d/1iCmdHud7Y296SpeOan0vNNJEsg4qBsin/view?usp=sharing. The file was generated using the query:
+```sql
+SELECT concept_name,
+       domain_id,
+       vocabulary_id,
+       concept_class_id,
+       standard_concept,
+       concept_code,
+       valid_start_date,
+       valid_end_date,
+       invalid_reason
+FROM concept_manual
+ORDER BY vocabulary_id, concept_code, invalid_reason, valid_start_date, valid_end_date, concept_name;
+```
+2.Extract the following csv file into the concept_synonym_manual table: https://drive.google.com/file/d/1C9qVJwR369y9Jk02iS-qK45Gn5iGGRJr/view?usp=sharing. The file was generated using the query:
+```sql
+SELECT synonym_name,
+       synonym_concept_code,
+       synonym_vocabulary_id,
+       language_concept_id
+FROM concept_synonym_manual
+ORDER BY synonym_vocabulary_id, synonym_concept_code, language_concept_id, synonym_name;`
+```
+3.Extract the following csv file into the concept_relationship_manual table: https://drive.google.com/file/d/1uUrkcknqIogs4Os0Hqu51WjSPo0C5mvH/view?usp=sharing. The file was generated using the query:
+```sql
+SELECT concept_code_1,
+       concept_code_2,
+       vocabulary_id_1,
+       vocabulary_id_2,
+       relationship_id,
+       valid_start_date,
+       valid_end_date,
+       invalid_reason
+FROM concept_relationship_manual
+ORDER BY vocabulary_id_1, vocabulary_id_2, relationship_id, concept_code_1, concept_code_2, invalid_reason, valid_start_date, valid_end_date;
+```
+##### csv format:
+- delimiter: ','
+- encoding: 'UTF8'
+- header: ON
+- decimal symbol: '.'
+- quote escape: with backslash \
+- quote always: FALSE
+- NULL string: empty

ICD10CM/readme.md

@@ -1,16 +1,19 @@
-Update of ICD10CM
+### ICD10CM upload / update ###
 
-Prerequisites:
-- Schema DevV5 with copies of tables concept, concept_relationship and concept_synonym from ProdV5, fully indexed. 
-- Schema UMLS
-- SNOMED must be loaded first
-- Working directory ICD10CM.
+#### Prerequisites: #### 
+* Schema DevV5 with copies of tables concept, concept_relationship and concept_synonym from ProdV5, fully indexed. 
+* Schema UMLS
+* SNOMED must be loaded first
+* Working directory, e.g. dev_icd10cm
 
-1. Run create_source_tables.sql
-2. Download ICD10CM_FYYYYY_code_descriptions.zip or icd10cm_order_YYYY.txt from ftp://ftp.cdc.gov/pub/Health_Statistics/NCHS/Publications/ICD10CM/YYYY
-3. Extract icd10cm_order_YYYY.txt (if needed) and rename to icd10cm.txt
-4. Run in devv5 (with fresh vocabulary date and version, e.g. according "The 2018 ICD-10-CM codes are to be used from October 1, 2017 through September 30, 2018" so 20171001): 
+#### Sequence of actions ####
+* Run create_source_tables.sql
+* Download ICD10CM_FYYYYY_code_descriptions.zip or icd10cm_order_YYYY.txt from ftp://ftp.cdc.gov/pub/Health_Statistics/NCHS/Publications/ICD10CM/YYYY
+* Extract icd10cm_order_YYYY.txt (if needed) and rename to icd10cm.txt
+* Perform actions described in the folder of 'manual_work'
+* Run in devv5 (with fresh vocabulary date and version, e.g. according "The 2018 ICD-10-CM codes are to be used from October 1, 2017 through September 30, 2018" so 20171001): 
+```sql
 SELECT sources.load_input_tables('ICD10CM',TO_DATE('20170428','YYYYMMDD'),'ICD10CM FY2017 code descriptions');
-5. Run load_stage.sql
-Note: Load_stage generates list of the relationships that need to be checked and modified by the medical coder and after uploading this data to server load_stage continues making proper relationships using this manually created table
-6. Run generic_update: devv5.GenericUpdate();
+```
+* Run load_stage.sql. Note: Load_stage generates list of the relationships that need to be checked and modified by the medical coder and after uploading this data to server load_stage continues making proper relationships using this manually created table
+* Run generic_update: devv5.GenericUpdate();