Skip to content

Commit

Permalink
Add samples field to custom filters to allow fetching samples without…
Browse files Browse the repository at this point in the history 
… session (#10998)

* add samples field and handling to custom filters

---------

Co-authored-by: Bryan Lai <[email protected]>
Co-authored-by: Zhaoyuan (Ryan) Fu <[email protected]>
  • Loading branch information
@gblaih @fuzhaoyuan
3 people authored Oct 2, 2024
1 parent f7d91c0 commit 857329b
Show file tree
Hide file tree
Showing 7 changed files with 2,691 additions and 15 deletions.
123 changes: 120 additions & 3 deletions src/main/java/org/cbioportal/web/columnar/util/CustomDataFilterUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;

@Component
public class CustomDataFilterUtil {
Expand All @@ -41,12 +42,28 @@ public List<CustomSampleIdentifier> extractCustomDataSamples(final StudyViewFilt
return null;
}

List<CustomSampleIdentifier> customSampleIdentifiers = new ArrayList<>();

if (CollectionUtils.isEmpty(studyViewFilter.getCustomDataFilters())) {
return null;
}

final List<CustomSampleIdentifier> customSamplesFromProperty = studyViewFilter.getCustomDataFilters().stream()
.flatMap(filter -> {
List<CustomSampleIdentifier> samples = filter.getSamples();
return (samples != null) ? samples.stream() : Stream.empty();
})
.toList();

if (!customSamplesFromProperty.isEmpty()) {
return extractCustomDataSamplesWithoutSession(studyViewFilter, customSamplesFromProperty);
}
else {
return extractCustomDataSamplesWithSession(studyViewFilter);
}
}

private List<CustomSampleIdentifier> extractCustomDataSamplesWithSession(final StudyViewFilter studyViewFilter) {
List<CustomSampleIdentifier> customSampleIdentifiers = new ArrayList<>();

final List<String> attributeIds = studyViewFilter.getCustomDataFilters().stream()
.map(ClinicalDataFilter::getAttributeId)
.collect(Collectors.toList());
Expand Down Expand Up @@ -74,7 +91,6 @@ public List<CustomSampleIdentifier> extractCustomDataSamples(final StudyViewFilt
CustomSampleIdentifier customSampleIdentifier = new CustomSampleIdentifier();
customSampleIdentifier.setStudyId(datum.getStudyId());
customSampleIdentifier.setSampleId(datum.getSampleId());
customSampleIdentifier.setAttributeId(customDataSession.getId());
customSampleIdentifiers.add(customSampleIdentifier);
customDataByStudySampleSession.put(datum.getStudyId(), datum.getSampleId(), customDataSession.getId(), value);
})
Expand Down Expand Up @@ -119,6 +135,74 @@ public List<CustomSampleIdentifier> extractCustomDataSamples(final StudyViewFilt

return filtered;
}

private List<CustomSampleIdentifier> extractCustomDataSamplesWithoutSession(final StudyViewFilter studyViewFilter, List<CustomSampleIdentifier> customSamplesFromProperty) {
List<CustomSampleIdentifier> customSampleIdentifiers = new ArrayList<>(customSamplesFromProperty);

List<ClinicalDataFilter> equalityFilters = new ArrayList<>();
List<ClinicalDataFilter> intervalFilters = new ArrayList<>();

studyViewFilter.getCustomDataFilters().forEach(filter -> {
if (filter.getDatatype()
.equals(CustomDatatype.STRING.name())
) {
equalityFilters.add(filter);
} else {
intervalFilters.add(filter);
}
});

MultiKeyMap<String, String> customDataByStudySampleName = new MultiKeyMap<>();

studyViewFilter.getCustomDataFilters().stream().forEach(filter ->
filter.getSamples().forEach(datum -> {
String value = datum.getValue().toUpperCase();
if (value.equals("NAN") || value.equals("N/A")) {
value = "NA";
}
customDataByStudySampleName.put(datum.getStudyId(), datum.getSampleId(), filter.getDisplayName(), value);
})
);

List<CustomSampleIdentifier> filtered = new ArrayList<>();
customSampleIdentifiers.forEach(customSampleIdentifier -> {
int equalityFilterCount = getFilteredCountByDataEqualityWithStudySampleNameMap(equalityFilters, customDataByStudySampleName,
customSampleIdentifier.getSampleId(), customSampleIdentifier.getStudyId(), false);
int intervalFilterCount = getFilteredCountByDataIntervalWithStudySampleNameMap(intervalFilters, customDataByStudySampleName,
customSampleIdentifier.getSampleId(), customSampleIdentifier.getStudyId());
if (equalityFilterCount == equalityFilters.size()
&& intervalFilterCount == intervalFilters.size()
) {
filtered.add(customSampleIdentifier);
}
else {
customSampleIdentifier.setIsFilteredOut(true);
filtered.add(customSampleIdentifier);
}
});
return filtered;
}

private Integer getFilteredCountByDataEqualityWithStudySampleNameMap(List<ClinicalDataFilter> attributes, MultiKeyMap<String, String> clinicalDataMap,
String entityId, String studyId, boolean negateFilters) {
Integer count = 0;
for (ClinicalDataFilter s : attributes) {
List<String> filteredValues = s.getValues()
.stream()
.map(DataFilterValue::getValue)
.collect(Collectors.toList());
filteredValues.replaceAll(String::toUpperCase);
if (clinicalDataMap.containsKey(studyId, entityId, s.getDisplayName())) {
String value = clinicalDataMap.get(studyId, entityId, s.getDisplayName());
if (negateFilters ^ filteredValues.contains(value)) {
count++;
}
} else if (negateFilters ^ filteredValues.contains("NA")) {
count++;
}
}
return count;
}

private <S> Integer getFilteredCountByDataInterval(List<ClinicalDataFilter> attributes, MultiKeyMap<String, S> clinicalDataMap,
String entityId, String studyId) {
Expand Down Expand Up @@ -157,6 +241,39 @@ else if (specialValues.contains(attrValue.toUpperCase())) {
return count;
}

private Integer getFilteredCountByDataIntervalWithStudySampleNameMap(List<ClinicalDataFilter> attributes, MultiKeyMap<String, String> clinicalDataMap,
String entityId, String studyId) {
int count = 0;

for (ClinicalDataFilter filter : attributes) {
if (clinicalDataMap.containsKey(studyId, entityId, filter.getDisplayName())) {
String attrValue = clinicalDataMap.get(studyId, entityId, filter.getDisplayName());
Range<BigDecimal> rangeValue = calculateRangeValueForAttr(attrValue);

// find range filters
List<Range<BigDecimal>> ranges = filter.getValues().stream()
.map(this::calculateRangeValueForFilter)
.filter(Objects::nonNull)
.toList();

// find special value filters
List<String> specialValues = filter.getValues().stream()
.filter(f -> f.getValue() != null)
.map(f -> f.getValue().toUpperCase())
.toList();

if ((rangeValue != null && ranges.stream().anyMatch(r -> r.encloses(rangeValue))) ||
(rangeValue == null && specialValues.contains(attrValue.toUpperCase()))) {
count++;
}
} else if (Boolean.TRUE.equals(containsNA(filter))) {
count++;
}
}

return count;
}

private Range<BigDecimal> calculateRangeValueForAttr(String attrValue) {
if (attrValue == null) {
return null;
Expand Down
27 changes: 27 additions & 0 deletions src/main/java/org/cbioportal/web/parameter/ClinicalDataFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
public class ClinicalDataFilter extends DataFilter implements Serializable {

private String attributeId;
private List<CustomSampleIdentifier> samples;
private String datatype;
private String displayName;

public String getAttributeId() {
return attributeId;
Expand All @@ -15,4 +18,28 @@ public void setAttributeId(String attributeId) {
this.attributeId = attributeId;
}

public List<CustomSampleIdentifier> getSamples() {
return samples;
}

public void setSamples(List<CustomSampleIdentifier> samples) {
this.samples = samples;
}

public String getDatatype() {
return datatype;
}

public void setDatatype(String datatype) {
this.datatype = datatype;
}

public String getDisplayName() {
return displayName;
}

public void setDisplayName(String displayName) {
this.displayName = displayName;
}

}
10 changes: 5 additions & 5 deletions src/main/java/org/cbioportal/web/parameter/CustomSampleIdentifier.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
public class CustomSampleIdentifier extends SampleIdentifier implements Serializable {

private boolean isFilteredOut = false;
private String attributeId;
private String value;

public boolean getIsFilteredOut() {
return isFilteredOut;
Expand All @@ -15,11 +15,11 @@ public void setIsFilteredOut(boolean isFilteredOut) {
this.isFilteredOut = isFilteredOut;
}

public String getAttributeId() {
return attributeId;
public String getValue() {
return value;
}

public void setAttributeId(String attributeId) {
this.attributeId = attributeId;
public void setValue(String value) {
this.value = value;
}
}
6 changes: 2 additions & 4 deletions src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@
sample_unique_id IN (
'',
<foreach item="sampleIdentifier" collection="studyViewFilterHelper.customDataSamples" separator=",">
<if test="customDataFilter.attributeId == sampleIdentifier.getAttributeId() and !sampleIdentifier.getIsFilteredOut()">
<if test="!sampleIdentifier.getIsFilteredOut()">
'${sampleIdentifier.studyId}_${sampleIdentifier.sampleId}'
</if>
</foreach>
Expand All @@ -94,9 +94,7 @@
OR
sample_unique_id NOT IN (
<foreach item="sampleIdentifier" collection="studyViewFilterHelper.customDataSamples" separator=",">
<if test="customDataFilter.attributeId == sampleIdentifier.getAttributeId()">
'${sampleIdentifier.studyId}_${sampleIdentifier.sampleId}'
</if>
'${sampleIdentifier.studyId}_${sampleIdentifier.sampleId}'
</foreach>
)
</if>
Expand Down
14 changes: 11 additions & 3 deletions src/test/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapperTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@
import org.cbioportal.persistence.helper.AlterationFilterHelper;
import org.cbioportal.persistence.helper.StudyViewFilterHelper;
import org.cbioportal.persistence.mybatisclickhouse.config.MyBatisConfig;
import org.cbioportal.web.parameter.*;
import org.cbioportal.web.parameter.ClinicalDataFilter;
import org.cbioportal.web.parameter.CustomSampleIdentifier;
import org.cbioportal.web.parameter.DataFilter;
import org.cbioportal.web.parameter.DataFilterValue;
import org.cbioportal.web.parameter.StudyViewFilter;
import org.cbioportal.web.parameter.filter.AndedPatientTreatmentFilters;
import org.cbioportal.web.parameter.filter.AndedSampleTreatmentFilters;
import org.cbioportal.web.parameter.filter.OredPatientTreatmentFilters;
Expand All @@ -23,7 +27,12 @@
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;

import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
Expand Down Expand Up @@ -59,7 +68,6 @@ public void getFilteredSamples() {
assertEquals(0, filteredSamples1.size());

CustomSampleIdentifier customSampleIdentifier = new CustomSampleIdentifier();
customSampleIdentifier.setAttributeId("123");
customSampleIdentifier.setStudyId("acc_tcga");
customSampleIdentifier.setSampleId("tcga-a1-a0sb-01");
var filteredSamples2 = studyViewMapper.getFilteredSamples(StudyViewFilterHelper.build(studyViewFilter, null, Arrays.asList(customSampleIdentifier)));
Expand Down
Loading

0 comments on commit 857329b

Please sign in to comment.