From 4e2ac9bdef97175f5d762c6e7f065a83512a3c40 Mon Sep 17 00:00:00 2001 From: AndreaGuarracino Date: Mon, 6 Jul 2020 15:24:06 +0200 Subject: renamed sra script; added seq technology in its additional information field if the term … --- .../SraExperimentPackage.2020.07.05.xml.gz | Bin 0 -> 6502056 bytes scripts/create_sra_metadata/create_sra_metadata.py | 247 +++++++++++++++++++++ .../SraExperimentPackage.2020.07.05.xml.gz | Bin 6502056 -> 0 bytes scripts/download_sra_data/download_sra_data.py | 243 -------------------- 4 files changed, 247 insertions(+), 243 deletions(-) create mode 100644 scripts/create_sra_metadata/SraExperimentPackage.2020.07.05.xml.gz create mode 100644 scripts/create_sra_metadata/create_sra_metadata.py delete mode 100644 scripts/download_sra_data/SraExperimentPackage.2020.07.05.xml.gz delete mode 100644 scripts/download_sra_data/download_sra_data.py (limited to 'scripts') diff --git a/scripts/create_sra_metadata/SraExperimentPackage.2020.07.05.xml.gz b/scripts/create_sra_metadata/SraExperimentPackage.2020.07.05.xml.gz new file mode 100644 index 0000000..88acb18 Binary files /dev/null and b/scripts/create_sra_metadata/SraExperimentPackage.2020.07.05.xml.gz differ diff --git a/scripts/create_sra_metadata/create_sra_metadata.py b/scripts/create_sra_metadata/create_sra_metadata.py new file mode 100644 index 0000000..470980e --- /dev/null +++ b/scripts/create_sra_metadata/create_sra_metadata.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 + +import os +from dateutil.parser import parse +import xml.etree.ElementTree as ET +import json +import gzip + +dir_yaml = 'yaml' + +date = '2020.07.05' + +# Query on SRA: 'txid2697049[Organism]' (https://www.ncbi.nlm.nih.gov/sra/?term=txid2697049%5BOrganism%5D) +# Query on SRA: 'txid2697049[Organism:noexp] NOT 0[Mbases ' (https://www.ncbi.nlm.nih.gov/sra/?term=txid2697049%5BOrganism:noexp%5D%20NOT%200[Mbases) +# -> Send to -> File -> Full XML -> Create File +path_sra_metadata_xml = 'SraExperimentPackage.{}.xml.gz'.format(date) + +dir_dict_ontology_standardization = '../dict_ontology_standardization/' +path_sra_study_accessions_txt = 'SRAStudyAccessions.{}.txt'.format(date) + +term_to_uri_dict = {} + +for path_dict_xxx_csv in [os.path.join(dir_dict_ontology_standardization, name_xxx_csv) for name_xxx_csv in os.listdir(dir_dict_ontology_standardization) if name_xxx_csv.endswith('.csv')]: + print('Read {}'.format(path_dict_xxx_csv)) + + with open(path_dict_xxx_csv, 'r') as f: + for line in f: + if len(line.split(',')) > 2: + term, uri = line.strip('\n').split('",') + term = term.strip('"') + else: + term, uri = line.strip('\n').split(',') + + term_to_uri_dict[term] = uri + +def is_integer(string_to_check): + try: + int(string_to_check) + return True + except ValueError: + return False + +if not os.path.exists(dir_yaml): + os.makedirs(dir_yaml) + +sra_metadata_xml_file = gzip.open(path_sra_metadata_xml, 'r') +tree = ET.parse(sra_metadata_xml_file) +sra_metadata_xml_file.close() + +EXPERIMENT_PACKAGE_SET = tree.getroot() + +missing_value_list = [] + +run_accession_set = set() +run_accession_to_downloadble_file_url_dict = {} + +for i, EXPERIMENT_PACKAGE in enumerate(EXPERIMENT_PACKAGE_SET): + #print(i, EXPERIMENT_PACKAGE) + + # A general default-empty yaml could be read from the definitive one + info_for_yaml_dict = { + 'id': 'placeholder', + 'host': {}, + 'sample': {}, + 'virus': {}, + 'technology': {}, + 'submitter': {} + } + + RUN_SET = EXPERIMENT_PACKAGE.find('RUN_SET') + RUN = RUN_SET.find('RUN') + accession = RUN.attrib['accession'] + run_accession_set.add(accession) + #print(accession) + + info_for_yaml_dict['sample']['sample_id'] = accession + + #SRAFiles = RUN.find('SRAFiles') + #if SRAFiles is not None: + # url = SRAFiles.find('SRAFile').attrib['url'] + # if 'sra-download.ncbi.nlm.nih.gov' in url: + # run_accession_to_downloadble_file_url_dict[accession] = url + + + SAMPLE = EXPERIMENT_PACKAGE.find('SAMPLE') + SAMPLE_ATTRIBUTE_list = SAMPLE.iter('SAMPLE_ATTRIBUTE') + + for SAMPLE_ATTRIBUTE in SAMPLE_ATTRIBUTE_list: + VALUE = SAMPLE_ATTRIBUTE.find('VALUE') + if VALUE is not None: + TAG_text = SAMPLE_ATTRIBUTE.find('TAG').text + VALUE_text = VALUE.text + + if TAG_text in ['host', 'host scientific name']: + if VALUE_text.lower() in ['homo sapien', 'homosapiens']: + VALUE_text = 'Homo sapiens' + + if VALUE_text in term_to_uri_dict: + info_for_yaml_dict['host']['host_species'] = term_to_uri_dict[VALUE_text] + else: + missing_value_list.append('\t'.join([accession, 'host_species', VALUE_text])) + elif TAG_text in ['host_health_status', 'host health state']: + if VALUE_text in term_to_uri_dict: + info_for_yaml_dict['host']['host_health_status'] = term_to_uri_dict[VALUE_text] + elif VALUE_text.strip("'") not in ['missing', 'not collected', 'not provided']: + missing_value_list.append('\t'.join([accession, 'host_health_status', VALUE_text])) + elif TAG_text in ['strain', 'isolate']: + if VALUE_text.lower() not in ['not applicable', 'missing', 'na', 'unknown', 'not provided']: + value_to_insert = VALUE_text + + if value_to_insert.lower() in ['homo sapien', 'homosapiens']: + value_to_insert = 'Homo sapiens' + + if value_to_insert in term_to_uri_dict: + value_to_insert = term_to_uri_dict[value_to_insert] + + if 'virus_strain' not in info_for_yaml_dict: + info_for_yaml_dict['virus']['virus_strain'] = value_to_insert + else: + info_for_yaml_dict['virus']['virus_strain'] += '; ' + value_to_insert + elif TAG_text in ['isolation_source', 'isolation source host-associated']: + if VALUE_text in term_to_uri_dict: + info_for_yaml_dict['sample']['specimen_source'] = [term_to_uri_dict[VALUE_text]] + else: + if VALUE_text.lower() in ['np/op', 'np/op swab', 'np/np swab', 'nasopharyngeal and oropharyngeal swab', 'nasopharyngeal/oropharyngeal swab', 'combined nasopharyngeal and oropharyngeal swab']: + info_for_yaml_dict['sample']['specimen_source'] = [term_to_uri_dict['nasopharyngeal swab'], term_to_uri_dict['oropharyngeal swab']] + elif VALUE_text.lower() in ['nasopharyngeal swab/throat swab', 'nasopharyngeal/throat swab', 'nasopharyngeal swab and throat swab', 'nasal swab and throat swab', 'nasopharyngeal aspirate/throat swab']: + info_for_yaml_dict['sample']['specimen_source'] = [term_to_uri_dict['nasopharyngeal swab'], term_to_uri_dict['throat swab']] + elif VALUE_text.lower() in ['nasal swab and throat swab']: + info_for_yaml_dict['sample']['specimen_source'] = [term_to_uri_dict['nasal swab'], term_to_uri_dict['throat swab']] + elif VALUE_text.lower() in ['nasal-swab and oro-pharyngeal swab']: + info_for_yaml_dict['sample']['specimen_source'] = [term_to_uri_dict['nasal swab'], term_to_uri_dict['oropharyngeal swab']] + elif VALUE_text.strip("'") not in ['missing', 'not collected', 'unknown', 'not provided', 'not applicable', 'N/A']: + missing_value_list.append('\t'.join([accession, 'specimen_source', VALUE_text])) + elif TAG_text in ['host_sex', 'host sex']: + if VALUE_text.lower() not in ['missing', 'not provided']: + if VALUE_text in ['male', 'female']: + info_for_yaml_dict['host']['host_sex'] = "http://purl.obolibrary.org/obo/PATO_0000384" if VALUE_text == 'male' else "http://purl.obolibrary.org/obo/PATO_0000383" + else: + missing_value_list.append('\t'.join([accession, 'host_sex', VALUE_text])) + elif TAG_text in ['host_age', 'host age']: + if is_integer(VALUE_text): + info_for_yaml_dict['host']['host_age'] = VALUE_text + info_for_yaml_dict['host']['host_age_unit'] = 'http://purl.obolibrary.org/obo/UO_0000036' + elif TAG_text == 'collected_by': + if VALUE_text.lower() not in ['not available', 'missing']: + name = VALUE_text in ['Dr. Susie Bartlett', 'Ahmed Babiker', 'Aisi Fu', 'Brandi Williamson', 'George Taiaroa', 'Natacha Ogando', 'Tim Dalebout', 'ykut Ozdarendeli'] + + info_for_yaml_dict['sample']['collector_name' if name else 'collecting_institution'] = VALUE_text + elif TAG_text == 'collecting institution': + if VALUE_text.lower() not in ['not provided', 'na']: + info_for_yaml_dict['sample']['collecting_institution'] = VALUE_text + elif TAG_text in ['collection_date', 'collection date']: + if VALUE_text.lower() not in ['not applicable', 'missing', 'na']: + date_to_write = VALUE_text + date_is_estimated = True + + VALUE_text_list = VALUE_text.split('-') + if len(VALUE_text_list) == 3: + date_is_estimated = False + + if VALUE_text_list[1].isalpha(): + date_to_write = parse(VALUE_text).strftime('%Y-%m-%d') + elif len(VALUE_text_list) == 2: + date_to_write = VALUE_text + '-15' + else: + if int(VALUE_text) < 2020: + date_to_write = "{}-12-15".format(VALUE_text) + else: + date_to_write = "{}-01-15".format(VALUE_text) + + info_for_yaml_dict['sample']['collection_date'] = date_to_write + + if date_is_estimated: + if 'additional_collection_information' in info_for_yaml_dict['sample']: + info_for_yaml_dict['sample']['additional_collection_information'] += "; The 'collection_date' is estimated (the original date was: {})".format(VALUE_text) + else: + info_for_yaml_dict['sample']['additional_collection_information'] = "The 'collection_date' is estimated (the original date was: {})".format(VALUE_text) + elif TAG_text == 'geo_loc_name': + if VALUE_text in term_to_uri_dict: + info_for_yaml_dict['sample']['collection_location'] = term_to_uri_dict[VALUE_text] + elif VALUE_text.lower() not in ['na', 'not applicable']: + missing_value_list.append('\t'.join([accession, 'geo_loc_name', VALUE_text])) + #else: + # if TAG_text not in ['lat_lon', 'host_disease', 'BioSampleModel', 'passage_history']: + # print(accession, TAG_text, VALUE_text) + + + taxon_id = SAMPLE.find('SAMPLE_NAME').find('TAXON_ID').text + info_for_yaml_dict['virus']['virus_species'] = "http://purl.obolibrary.org/obo/NCBITaxon_"+taxon_id + + + EXPERIMENT = EXPERIMENT_PACKAGE.find('EXPERIMENT') + INSTRUMENT_MODEL = [x.text for x in EXPERIMENT.find('PLATFORM').iter('INSTRUMENT_MODEL')][0] + + if INSTRUMENT_MODEL.lower() != 'unspecified': + if INSTRUMENT_MODEL in term_to_uri_dict: + info_for_yaml_dict['technology']['sample_sequencing_technology'] = [term_to_uri_dict[INSTRUMENT_MODEL]] + else: + info_for_yaml_dict['technology']['additional_technology_information'] = INSTRUMENT_MODEL + missing_value_list.append('\t'.join([accession, 'sample_sequencing_technology', INSTRUMENT_MODEL])) + #else: + # print(accession, 'Missing INSTRUMENT_MODEL', info_for_yaml_dict) + LIBRARY_DESCRIPTOR = EXPERIMENT.find('DESIGN').find('LIBRARY_DESCRIPTOR') + if LIBRARY_DESCRIPTOR.text not in ['OTHER']: + info_for_yaml_dict['technology']['additional_technology_information'] = 'LIBRARY_STRATEGY: {};'.format(LIBRARY_DESCRIPTOR.find('LIBRARY_STRATEGY').text) + + SUBMISSION = EXPERIMENT_PACKAGE.find('SUBMISSION') + info_for_yaml_dict['submitter']['submitter_sample_id'] = SUBMISSION.attrib['accession'] + + if SUBMISSION.attrib['lab_name'].lower() not in ['na']: + info_for_yaml_dict['submitter']['originating_lab'] = SUBMISSION.attrib['lab_name'] + + STUDY = EXPERIMENT_PACKAGE.find('STUDY') + info_for_yaml_dict['submitter']['publication'] = STUDY.attrib['alias'] + + + Organization = EXPERIMENT_PACKAGE.find('Organization') + Organization_Name = Organization.find('Name') + info_for_yaml_dict['submitter']['authors'] = [Organization_Name.text] + + Organization_Contact = Organization.find('Contact') + if Organization_Contact is not None: + Organization_Contact_Name = Organization_Contact.find('Name') + info_for_yaml_dict['submitter']['submitter_name'] = [Organization_Contact_Name.find('First').text + ' ' + Organization_Contact_Name.find('Last').text] + info_for_yaml_dict['submitter']['additional_submitter_information'] = Organization_Contact.attrib['email'] + + Organization_Concact_Address = Organization_Contact.find('Address') + if Organization_Concact_Address is not None: + info_for_yaml_dict['submitter']['submitter_address'] = '; '.join([x.text for x in Organization_Concact_Address] + ['Postal code ' + Organization_Concact_Address.attrib['postal_code']]) + + Organization_Address = Organization.find('Address') + if Organization_Address is not None: + info_for_yaml_dict['submitter']['lab_address'] = '; '.join([x.text for x in Organization_Address] + ['Postal code ' + Organization_Address.attrib['postal_code']]) + + if 'collection_date' not in info_for_yaml_dict['sample']: + info_for_yaml_dict['sample']['collection_date'] = '1970-01-01' + info_for_yaml_dict['sample']['additional_collection_information'] = "The real 'collection_date' is missing" + + with open(os.path.join(dir_yaml, '{}.yaml'.format(accession)), 'w') as fw: + json.dump(info_for_yaml_dict, fw, indent=2) + +if len(missing_value_list) > 0: + path_missing_terms_tsv = 'missing_terms.tsv' + print('Written missing terms in {}'.format(path_missing_terms_tsv)) + with open(path_missing_terms_tsv, 'w') as fw: + fw.write('\n'.join(missing_value_list)) diff --git a/scripts/download_sra_data/SraExperimentPackage.2020.07.05.xml.gz b/scripts/download_sra_data/SraExperimentPackage.2020.07.05.xml.gz deleted file mode 100644 index 88acb18..0000000 Binary files a/scripts/download_sra_data/SraExperimentPackage.2020.07.05.xml.gz and /dev/null differ diff --git a/scripts/download_sra_data/download_sra_data.py b/scripts/download_sra_data/download_sra_data.py deleted file mode 100644 index e36afb6..0000000 --- a/scripts/download_sra_data/download_sra_data.py +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env python3 - -import os -from dateutil.parser import parse -import xml.etree.ElementTree as ET -import json -import gzip - -dir_yaml = 'yaml' - -date = '2020.07.05' - -# Query on SRA: 'txid2697049[Organism]' (https://www.ncbi.nlm.nih.gov/sra/?term=txid2697049%5BOrganism%5D) -# Query on SRA: 'txid2697049[Organism:noexp] NOT 0[Mbases ' (https://www.ncbi.nlm.nih.gov/sra/?term=txid2697049%5BOrganism:noexp%5D%20NOT%200[Mbases) -# -> Send to -> File -> Full XML -> Create File -path_sra_metadata_xml = 'SraExperimentPackage.{}.xml.gz'.format(date) - -dir_dict_ontology_standardization = '../dict_ontology_standardization/' -path_sra_study_accessions_txt = 'SRAStudyAccessions.{}.txt'.format(date) - -term_to_uri_dict = {} - -for path_dict_xxx_csv in [os.path.join(dir_dict_ontology_standardization, name_xxx_csv) for name_xxx_csv in os.listdir(dir_dict_ontology_standardization) if name_xxx_csv.endswith('.csv')]: - print('Read {}'.format(path_dict_xxx_csv)) - - with open(path_dict_xxx_csv, 'r') as f: - for line in f: - if len(line.split(',')) > 2: - term, uri = line.strip('\n').split('",') - term = term.strip('"') - else: - term, uri = line.strip('\n').split(',') - - term_to_uri_dict[term] = uri - -def is_integer(string_to_check): - try: - int(string_to_check) - return True - except ValueError: - return False - -if not os.path.exists(dir_yaml): - os.makedirs(dir_yaml) - -sra_metadata_xml_file = gzip.open(path_sra_metadata_xml, 'r') -tree = ET.parse(sra_metadata_xml_file) -sra_metadata_xml_file.close() - -EXPERIMENT_PACKAGE_SET = tree.getroot() - -missing_value_list = [] - -run_accession_set = set() -run_accession_to_downloadble_file_url_dict = {} - -for i, EXPERIMENT_PACKAGE in enumerate(EXPERIMENT_PACKAGE_SET): - #print(i, EXPERIMENT_PACKAGE) - - # A general default-empty yaml could be read from the definitive one - info_for_yaml_dict = { - 'id': 'placeholder', - 'host': {}, - 'sample': {}, - 'virus': {}, - 'technology': {}, - 'submitter': {} - } - - RUN_SET = EXPERIMENT_PACKAGE.find('RUN_SET') - RUN = RUN_SET.find('RUN') - accession = RUN.attrib['accession'] - run_accession_set.add(accession) - #print(accession) - - info_for_yaml_dict['sample']['sample_id'] = accession - - #SRAFiles = RUN.find('SRAFiles') - #if SRAFiles is not None: - # url = SRAFiles.find('SRAFile').attrib['url'] - # if 'sra-download.ncbi.nlm.nih.gov' in url: - # run_accession_to_downloadble_file_url_dict[accession] = url - - - SAMPLE = EXPERIMENT_PACKAGE.find('SAMPLE') - SAMPLE_ATTRIBUTE_list = SAMPLE.iter('SAMPLE_ATTRIBUTE') - - for SAMPLE_ATTRIBUTE in SAMPLE_ATTRIBUTE_list: - VALUE = SAMPLE_ATTRIBUTE.find('VALUE') - if VALUE is not None: - TAG_text = SAMPLE_ATTRIBUTE.find('TAG').text - VALUE_text = VALUE.text - - if TAG_text in ['host', 'host scientific name']: - if VALUE_text in term_to_uri_dict: - info_for_yaml_dict['host']['host_species'] = term_to_uri_dict[VALUE_text] - else: - missing_value_list.append('\t'.join([accession, 'host_species', VALUE_text])) - elif TAG_text in ['host_health_status', 'host health state']: - if VALUE_text in term_to_uri_dict: - info_for_yaml_dict['host']['host_health_status'] = term_to_uri_dict[VALUE_text] - elif VALUE_text.strip("'") not in ['missing', 'not collected', 'not provided']: - missing_value_list.append('\t'.join([accession, 'host_health_status', VALUE_text])) - elif TAG_text in ['strain', 'isolate']: - if VALUE_text.lower() not in ['not applicable', 'missing', 'na', 'unknown', 'not provided']: - value_to_insert = VALUE_text - - if value_to_insert.lower() in ['homo sapien']: - value_to_insert = 'Homo sapiens' - - if value_to_insert in term_to_uri_dict: - value_to_insert = term_to_uri_dict[value_to_insert] - - if 'virus_strain' not in info_for_yaml_dict: - info_for_yaml_dict['virus']['virus_strain'] = value_to_insert - else: - info_for_yaml_dict['virus']['virus_strain'] += '; ' + value_to_insert - elif TAG_text in ['isolation_source', 'isolation source host-associated']: - if VALUE_text in term_to_uri_dict: - info_for_yaml_dict['sample']['specimen_source'] = [term_to_uri_dict[VALUE_text]] - else: - if VALUE_text.lower() in ['np/op', 'np/op swab', 'np/np swab', 'nasopharyngeal and oropharyngeal swab', 'nasopharyngeal/oropharyngeal swab', 'combined nasopharyngeal and oropharyngeal swab']: - info_for_yaml_dict['sample']['specimen_source'] = [term_to_uri_dict['nasopharyngeal swab'], term_to_uri_dict['oropharyngeal swab']] - elif VALUE_text.lower() in ['nasopharyngeal swab/throat swab', 'nasopharyngeal/throat swab', 'nasopharyngeal swab and throat swab', 'nasal swab and throat swab', 'nasopharyngeal aspirate/throat swab']: - info_for_yaml_dict['sample']['specimen_source'] = [term_to_uri_dict['nasopharyngeal swab'], term_to_uri_dict['throat swab']] - elif VALUE_text.lower() in ['nasal swab and throat swab']: - info_for_yaml_dict['sample']['specimen_source'] = [term_to_uri_dict['nasal swab'], term_to_uri_dict['throat swab']] - elif VALUE_text.lower() in ['nasal-swab and oro-pharyngeal swab']: - info_for_yaml_dict['sample']['specimen_source'] = [term_to_uri_dict['nasal swab'], term_to_uri_dict['oropharyngeal swab']] - elif VALUE_text.strip("'") not in ['missing', 'not collected', 'unknown', 'not provided', 'not applicable', 'N/A']: - missing_value_list.append('\t'.join([accession, 'specimen_source', VALUE_text])) - elif TAG_text in ['host_sex', 'host sex']: - if VALUE_text.lower() not in ['missing', 'not provided']: - if VALUE_text in ['male', 'female']: - info_for_yaml_dict['host']['host_sex'] = "http://purl.obolibrary.org/obo/PATO_0000384" if VALUE_text == 'male' else "http://purl.obolibrary.org/obo/PATO_0000383" - else: - missing_value_list.append('\t'.join([accession, 'host_sex', VALUE_text])) - elif TAG_text in ['host_age', 'host age']: - if is_integer(VALUE_text): - info_for_yaml_dict['host']['host_age'] = VALUE_text - info_for_yaml_dict['host']['host_age_unit'] = 'http://purl.obolibrary.org/obo/UO_0000036' - elif TAG_text == 'collected_by': - if VALUE_text.lower() not in ['not available', 'missing']: - name = VALUE_text in ['Dr. Susie Bartlett', 'Ahmed Babiker', 'Aisi Fu', 'Brandi Williamson', 'George Taiaroa', 'Natacha Ogando', 'Tim Dalebout', 'ykut Ozdarendeli'] - - info_for_yaml_dict['sample']['collector_name' if name else 'collecting_institution'] = VALUE_text - elif TAG_text == 'collecting institution': - if VALUE_text.lower() not in ['not provided', 'na']: - info_for_yaml_dict['sample']['collecting_institution'] = VALUE_text - elif TAG_text in ['collection_date', 'collection date']: - if VALUE_text.lower() not in ['not applicable', 'missing', 'na']: - date_to_write = VALUE_text - date_is_estimated = True - - VALUE_text_list = VALUE_text.split('-') - if len(VALUE_text_list) == 3: - date_is_estimated = False - - if VALUE_text_list[1].isalpha(): - date_to_write = parse(VALUE_text).strftime('%Y-%m-%d') - elif len(VALUE_text_list) == 2: - date_to_write = VALUE_text + '-15' - else: - if int(VALUE_text) < 2020: - date_to_write = "{}-12-15".format(VALUE_text) - else: - date_to_write = "{}-01-15".format(VALUE_text) - - info_for_yaml_dict['sample']['collection_date'] = date_to_write - - if date_is_estimated: - if 'additional_collection_information' in info_for_yaml_dict['sample']: - info_for_yaml_dict['sample']['additional_collection_information'] += "; The 'collection_date' is estimated (the original date was: {})".format(VALUE_text) - else: - info_for_yaml_dict['sample']['additional_collection_information'] = "The 'collection_date' is estimated (the original date was: {})".format(VALUE_text) - elif TAG_text == 'geo_loc_name': - if VALUE_text in term_to_uri_dict: - info_for_yaml_dict['sample']['collection_location'] = term_to_uri_dict[VALUE_text] - elif VALUE_text.lower() not in ['na', 'not applicable']: - missing_value_list.append('\t'.join([accession, 'geo_loc_name', VALUE_text])) - #else: - # if TAG_text not in ['lat_lon', 'host_disease', 'BioSampleModel', 'passage_history']: - # print(accession, TAG_text, VALUE_text) - - - taxon_id = SAMPLE.find('SAMPLE_NAME').find('TAXON_ID').text - info_for_yaml_dict['virus']['virus_species'] = "http://purl.obolibrary.org/obo/NCBITaxon_"+taxon_id - - - EXPERIMENT = EXPERIMENT_PACKAGE.find('EXPERIMENT') - INSTRUMENT_MODEL = [x.text for x in EXPERIMENT.find('PLATFORM').iter('INSTRUMENT_MODEL')][0] - - if INSTRUMENT_MODEL.lower() != 'unspecified': - if INSTRUMENT_MODEL in term_to_uri_dict: - info_for_yaml_dict['technology']['sample_sequencing_technology'] = [term_to_uri_dict[INSTRUMENT_MODEL]] - else: - missing_value_list.append('\t'.join([accession, 'sample_sequencing_technology', INSTRUMENT_MODEL])) - #else: - # print(accession, 'Missing INSTRUMENT_MODEL', info_for_yaml_dict) - LIBRARY_DESCRIPTOR = EXPERIMENT.find('DESIGN').find('LIBRARY_DESCRIPTOR') - if LIBRARY_DESCRIPTOR.text not in ['OTHER']: - info_for_yaml_dict['technology']['additional_technology_information'] = 'LIBRARY_STRATEGY: {};'.format(LIBRARY_DESCRIPTOR.find('LIBRARY_STRATEGY').text) - - SUBMISSION = EXPERIMENT_PACKAGE.find('SUBMISSION') - info_for_yaml_dict['submitter']['submitter_sample_id'] = SUBMISSION.attrib['accession'] - - if SUBMISSION.attrib['lab_name'].lower() not in ['na']: - info_for_yaml_dict['submitter']['originating_lab'] = SUBMISSION.attrib['lab_name'] - - STUDY = EXPERIMENT_PACKAGE.find('STUDY') - info_for_yaml_dict['submitter']['publication'] = STUDY.attrib['alias'] - - - Organization = EXPERIMENT_PACKAGE.find('Organization') - Organization_Name = Organization.find('Name') - info_for_yaml_dict['submitter']['authors'] = [Organization_Name.text] - - Organization_Contact = Organization.find('Contact') - if Organization_Contact is not None: - Organization_Contact_Name = Organization_Contact.find('Name') - info_for_yaml_dict['submitter']['submitter_name'] = [Organization_Contact_Name.find('First').text + ' ' + Organization_Contact_Name.find('Last').text] - info_for_yaml_dict['submitter']['additional_submitter_information'] = Organization_Contact.attrib['email'] - - Organization_Concact_Address = Organization_Contact.find('Address') - if Organization_Concact_Address is not None: - info_for_yaml_dict['submitter']['submitter_address'] = '; '.join([x.text for x in Organization_Concact_Address] + ['Postal code ' + Organization_Concact_Address.attrib['postal_code']]) - - Organization_Address = Organization.find('Address') - if Organization_Address is not None: - info_for_yaml_dict['submitter']['lab_address'] = '; '.join([x.text for x in Organization_Address] + ['Postal code ' + Organization_Address.attrib['postal_code']]) - - if 'collection_date' not in info_for_yaml_dict['sample']: - info_for_yaml_dict['sample']['collection_date'] = '1970-01-01' - info_for_yaml_dict['sample']['additional_collection_information'] = "The real 'collection_date' is missing" - - with open(os.path.join(dir_yaml, '{}.yaml'.format(accession)), 'w') as fw: - json.dump(info_for_yaml_dict, fw, indent=2) - -if len(missing_value_list) > 0: - path_missing_terms_tsv = 'missing_terms.tsv' - print('Written missing terms in {}'.format(path_missing_terms_tsv)) - with open(path_missing_terms_tsv, 'w') as fw: - fw.write('\n'.join(missing_value_list)) -- cgit v1.2.3