From 3503d925c5be9f24d9b0c4c0ce79fb0da338a13d Mon Sep 17 00:00:00 2001 From: Andrea Guarracino Date: Tue, 14 Apr 2020 18:51:01 +0200 Subject: Rename script/from_genbank_to_fasta_and_yaml.py to scripts/from_genbank_to_fasta_and_yaml.py --- scripts/from_genbank_to_fasta_and_yaml.py | 141 ++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 scripts/from_genbank_to_fasta_and_yaml.py (limited to 'scripts') diff --git a/scripts/from_genbank_to_fasta_and_yaml.py b/scripts/from_genbank_to_fasta_and_yaml.py new file mode 100644 index 0000000..0cc1a57 --- /dev/null +++ b/scripts/from_genbank_to_fasta_and_yaml.py @@ -0,0 +1,141 @@ +from Bio import Entrez +Entrez.email = 'your_email_to_be_polite' + +import xml.etree.ElementTree as ET +import yaml +import os + +path_ncbi_virus_accession = 'sequences.acc' + +date = '20200414' +path_seq_fasta = 'seq_from_nuccore.{}.fasta'.format(date) +path_metadata_xml = 'metadata_from_nuccore.{}.xml'.format(date) + +# Take all the ids +id_set = set() + +term_list = ['SARS-CoV-2', 'SARS-CoV2', 'SARS CoV2', 'SARSCoV2', 'txid2697049[Organism]'] +for term in term_list: + tmp_list = Entrez.read( + Entrez.esearch(db='nuccore', term=term, idtype='acc', retmax='10000') + )['IdList'] + print(term, len(tmp_list)) + + # Remove the version in the id + id_set.update([x.split('.')[0] for x in tmp_list]) + +print(term_list, len(id_set)) + +with open(path_ncbi_virus_accession) as f: + tmp_list = [line.strip('\n') for line in f] + +print('NCBI Virus', len(tmp_list)) +id_set.update(tmp_list) + +print(term_list + ['NCBI Virus'], len(id_set)) + +if not os.path.exists(path_metadata_xml): + # TO_DO: to check if I already have the records? + + with open(path_metadata_xml, 'w') as fw: + fw.write( + Entrez.efetch(db='nuccore', id=list(id_set), retmode='xml').read() + ) + + +tree = ET.parse(path_metadata_xml) +GBSet = tree.getroot() + +species_to_taxid_dict = { + 'Homo sapiens': 9606 +} + +for GBSeq in GBSet: + accession_version = GBSeq.find('GBSeq_accession-version').text + + GBSeq_sequence = GBSeq.find('GBSeq_sequence') + if GBSeq_sequence is None: + print(accession_version, ' - sequence not found') + continue + + + # A general default-empty yaml could be read from the definitive one + info_for_yaml_dict = { + 'id': 'placeholder', + 'host': {}, + 'sample': {}, + 'virus': {}, + 'technology': {}, + 'submitter': {} + } + + + info_for_yaml_dict['sample']['sample_id'] = accession_version + info_for_yaml_dict['submitter']['authors'] = ';'.join([x.text for x in GBSeq.iter('GBAuthor')]) + + + GBSeq_comment = GBSeq.find('GBSeq_comment') + if GBSeq_comment is not None and 'Assembly-Data' in GBSeq_comment.text: + GBSeq_comment_text = GBSeq_comment.text.split('##Assembly-Data-START## ; ')[1].split(' ; ##Assembly-Data-END##')[0] + + for info_to_check, field_in_yaml in zip( + ['Assembly Method', 'Coverage', 'Sequencing Technology'], + ['sequence_assembly_method', 'sequencing_coverage', 'sample_sequencing_technology'] + ): + if info_to_check in GBSeq_comment_text: + info_for_yaml_dict['technology'][field_in_yaml] = GBSeq_comment_text.split('{} :: '.format(info_to_check))[1].split(' ;')[0] + + + for GBFeature in GBSeq.iter('GBFeature'): + if GBFeature.find('GBFeature_key').text != 'source': + continue + + for GBQualifier in GBFeature.iter('GBQualifier'): + GBQualifier_value = GBQualifier.find('GBQualifier_value') + if GBQualifier_value is None: + continue + GBQualifier_value_text = GBQualifier_value.text + + GBQualifier_name_text = GBQualifier.find('GBQualifier_name').text + + if GBQualifier_name_text == 'host': + GBQualifier_value_text_list = GBQualifier_value_text.split('; ') + + info_for_yaml_dict['host']['host_common_name'] = GBQualifier_value_text_list[0] + + if GBQualifier_value_text_list[0] in species_to_taxid_dict: + info_for_yaml_dict['host']['host_species'] = species_to_taxid_dict[GBQualifier_value_text_list[0]] + + if len(GBQualifier_value_text_list) > 1: + if GBQualifier_value_text_list[1] in ['male', 'female']: + info_for_yaml_dict['host']['host_sex'] = GBQualifier_value_text_list[1] + else: + info_for_yaml_dict['host']['host_health_status'] = GBQualifier_value_text_list[1] + + if 'age' in GBQualifier_value_text: + info_for_yaml_dict['host']['host_age'] = int(GBQualifier_value_text_list[2].split('age ')[1]) + info_for_yaml_dict['host']['host_age_unit'] = 'year' + elif GBQualifier_name_text == 'collected_by': + if any([x in GBQualifier_value_text.lower() for x in ['institute', 'hospital', 'city', 'center']]): + info_for_yaml_dict['sample']['collecting_institution'] = GBQualifier_value_text + else: + info_for_yaml_dict['sample']['collector_name'] = GBQualifier_value_text + elif GBQualifier_name_text == 'isolation_source': + info_for_yaml_dict['sample']['specimen_source'] = GBQualifier_value_text + elif GBQualifier_name_text == 'collection_date': + # TO_DO: which format we will use? + info_for_yaml_dict['sample']['collection_date'] = GBQualifier_value_text + elif GBQualifier_name_text in ['lat_lon', 'country']: + info_for_yaml_dict['sample']['collection_location'] = GBQualifier_value_text + elif GBQualifier_name_text == 'note': + info_for_yaml_dict['sample']['additional_collection_information'] = GBQualifier_value_text + elif GBQualifier_name_text == 'isolate': + info_for_yaml_dict['virus']['virus_strain'] = GBQualifier_value_text + elif GBQualifier_name_text == 'db_xref': + info_for_yaml_dict['virus']['virus_species'] = int(GBQualifier_value_text.split('taxon:')[1]) + + with open('{}.fasta'.format(accession_version), 'w') as fw: + fw.write('>{}\n{}'.format(accession_version, GBSeq_sequence.text.upper())) + + with open('{}.yaml'.format(accession_version), 'w') as fw: + yaml.dump(info_for_yaml_dict, fw, default_flow_style=False) -- cgit v1.2.3 From 0152bd2efbced33368794092f4027cea064ffa44 Mon Sep 17 00:00:00 2001 From: Andrea Guarracino Date: Tue, 14 Apr 2020 18:51:59 +0200 Subject: accessions list CoV-2 from NCBI Virus 2020/04/14 --- scripts/sequences.acc | 877 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 877 insertions(+) create mode 100644 scripts/sequences.acc (limited to 'scripts') diff --git a/scripts/sequences.acc b/scripts/sequences.acc new file mode 100644 index 0000000..62bde2c --- /dev/null +++ b/scripts/sequences.acc @@ -0,0 +1,877 @@ +MT325599 +MT325601 +MT325602 +MT325607 +MT325608 +MT325609 +MT325610 +MT325612 +MT325616 +MT325617 +MT325618 +MT325622 +MT325623 +MT325600 +MT325606 +MT325611 +MT325613 +MT325615 +MT325619 +MT325620 +MT325624 +MT325625 +MT325565 +MT325566 +MT326147 +MT326153 +MT326173 +MT326174 +MT326176 +MT326186 +MT326023 +MT326026 +MT326027 +MT326034 +MT326039 +MT326042 +MT326045 +MT326047 +MT326051 +MT326054 +MT326060 +MT326061 +MT326064 +MT326065 +MT326068 +MT326072 +MT326076 +MT326078 +MT326079 +MT325590 +MT325640 +MT326130 +MT326129 +MT326128 +MT326121 +MT326119 +MT326109 +MT326100 +MT325568 +MT324679 +MT325561 +MT325571 +MT325585 +MT325587 +MT325588 +MT325589 +MT325596 +MT325597 +MT325603 +MT325614 +MT325621 +MT325629 +MT325630 +MT325638 +MT325639 +MT326086 +MT326087 +MT326102 +MT326104 +MT326105 +MT326123 +MT328033 +MT328034 +MT325562 +MT325564 +MT325567 +MT326154 +MT326155 +MT326156 +MT326157 +MT326163 +MT326165 +MT326175 +MT326177 +MT326184 +MT326185 +MT326187 +MT325572 +MT325575 +MT325583 +MT325584 +MT325604 +MT325631 +MT325632 +MT325635 +MT325636 +MT325637 +MT326095 +MT326096 +MT326103 +MT326112 +MT326113 +MT326114 +MT326115 +MT326122 +MT326131 +MT326132 +MT326133 +MT325563 +MT326164 +MT326166 +MT326167 +MT325569 +MT326097 +MT326106 +MT326107 +MT326116 +MT326117 +MT326124 +MT326125 +MT326126 +MT326127 +MT326134 +MT326135 +MT326136 +MT326137 +MT326138 +MT326139 +MT326140 +MT326141 +MT326142 +MT326143 +MT326144 +MT326145 +MT326146 +MT326148 +MT326149 +MT326150 +MT326151 +MT326152 +MT326158 +MT326159 +MT326160 +MT326161 +MT326162 +MT326168 +MT326169 +MT326170 +MT326171 +MT326172 +MT326178 +MT326179 +MT326180 +MT326181 +MT326182 +MT326183 +MT326188 +MT326189 +MT326190 +MT326191 +MT326120 +MT326118 +MT326111 +MT326110 +MT326108 +MT326101 +MT326099 +MT326098 +MT326094 +MT326093 +MT326092 +MT326091 +MT326090 +MT326085 +MT326084 +MT326083 +MT326082 +MT326081 +MT326080 +MT326077 +MT326067 +MT326057 +MT326024 +MT326025 +MT326032 +MT326033 +MT326035 +MT326036 +MT326037 +MT326040 +MT326041 +MT326043 +MT326044 +MT326046 +MT326049 +MT326050 +MT326052 +MT326053 +MT326055 +MT326056 +MT326059 +MT326062 +MT326063 +MT326066 +MT326069 +MT326070 +MT326071 +MT326073 +MT326074 +MT326075 +MT326088 +MT326089 +MT327745 +MT324062 +MT324680 +MT324684 +MT325573 +MT325574 +MT325576 +MT325577 +MT325578 +MT325580 +MT325591 +MT325592 +MT325593 +MT325595 +MT325605 +MT325627 +MT326028 +MT326029 +MT326031 +MT326048 +MT325570 +MT325579 +MT325581 +MT325582 +MT325586 +MT325594 +MT325598 +MT325626 +MT325628 +MT325633 +MT325634 +MT326030 +MT326038 +MT326058 +MT324681 +MT324682 +MT324683 +MT328032 +MT328035 +MT039874 +MT077125 +MT322394 +MT322397 +MT322398 +MT322399 +MT322400 +MT322401 +MT322403 +MT322404 +MT322405 +MT322406 +MT322408 +MT322409 +MT322410 +MT322411 +MT322412 +MT322413 +MT322414 +MT322415 +MT322416 +MT322417 +MT322418 +MT322419 +MT322420 +MT322421 +MT322422 +MT322423 +MT322424 +MT322396 +MT322402 +MT322395 +MT322407 +MT320538 +MT320891 +MT308692 +MT308693 +MT308698 +MT308699 +MT308703 +MT308704 +MT308694 +MT308695 +MT308696 +MT308697 +MT308700 +MT308701 +MT308702 +MT304476 +MT304474 +MT304475 +MT293547 +MT304477 +MT304483 +MT300186 +MT304478 +MT304479 +MT304480 +MT304481 +MT304482 +MT304484 +MT304485 +MT304486 +MT304487 +MT304488 +MT304489 +MT304490 +MT304491 +MT291831 +MT291836 +MT291834 +MT291835 +MT292570 +MT293173 +MT292574 +MT293179 +MT293181 +MT293183 +MT293195 +MT293196 +MT293201 +MT293204 +MT291829 +MT291830 +MT291827 +MT292572 +MT292577 +MT293186 +MT293187 +MT293188 +MT292580 +MT292581 +MT292571 +MT292576 +MT292578 +MT293185 +MT293160 +MT293161 +MT293199 +MT292579 +MT291828 +MT293166 +MT293167 +MT293168 +MT293175 +MT293190 +MT293191 +MT273658 +MT293159 +MT292582 +MT293162 +MT293163 +MT293164 +MT293165 +MT293156 +MT293157 +MT293158 +MT281577 +MT293171 +MT293174 +MT293176 +MT293182 +MT293210 +MT293211 +MT293217 +MT293218 +MT295465 +MT293213 +MT293221 +MT295464 +MT292569 +MT293169 +MT293172 +MT293177 +MT293200 +MT293198 +MT293205 +MT293207 +MT293212 +MT293216 +MT293219 +MT293222 +MT293224 +MT293225 +MT293206 +MT293208 +MT293209 +MT293214 +MT293215 +MT293220 +MT293170 +MT292573 +MT293178 +MT292575 +MT293180 +MT293184 +MT293189 +MT293192 +MT293193 +MT293194 +MT293197 +MT293202 +MT293203 +MT293223 +MT291826 +MT291832 +MT291833 +MT281530 +MT276331 +MT276325 +MT276324 +MT276326 +MT276327 +MT276330 +MT276329 +MT276597 +MT276598 +MT276323 +MT276328 +MT262896 +MT263385 +MT263392 +MT262900 +MT262901 +MT262902 +MT262909 +MT262911 +MT262912 +MT263382 +MT263383 +MT263384 +MT263423 +MT263431 +MT263421 +MT263443 +MT263461 +MT263420 +MT263429 +MT263434 +MT263435 +MT263437 +MT263445 +MT263428 +MT263433 +MT263436 +MT263438 +MT263440 +MT263444 +MT263452 +MT263455 +MT263456 +MT263462 +MT263463 +MT263466 +MT263446 +MT263447 +MT263448 +MT263449 +MT263451 +MT263453 +MT263388 +MT263391 +MT262903 +MT262906 +MT262907 +MT262908 +MT262913 +MT262914 +MT263390 +MT263398 +MT263403 +MT263430 +MT263399 +MT263404 +MT263405 +MT263414 +MT263389 +MT263393 +MT263394 +MT263395 +MT263396 +MT263397 +MT263402 +MT263469 +MT263441 +MT263454 +MT263467 +MT263465 +MT263468 +MT263439 +MT263457 +MT263460 +MT263450 +MT263458 +MT263459 +MT263464 +MT263386 +MT263387 +MT262897 +MT262899 +MT262904 +MT262905 +MT262910 +MT263408 +MT263412 +MT263416 +MT263417 +MT263422 +MT263432 +MT263419 +MT263424 +MT263427 +MT263442 +MT263413 +MT263418 +MT263425 +MT263401 +MT263409 +MT263410 +MT263411 +MT263426 +MT263406 +MT263407 +MT263415 +MT262993 +MT263074 +MT263381 +MT262898 +MT262915 +MT262916 +MT263400 +MT259257 +MT259261 +MT259262 +MT259263 +MT259264 +MT259268 +MT259269 +MT259270 +MT259271 +MT259272 +MT259273 +MT259275 +MT259276 +MT259277 +MT259279 +MT259244 +MT259245 +MT258381 +MT258377 +MT258379 +MT259226 +MT259281 +MT259282 +MT259283 +MT258378 +MT259231 +MT259274 +MT259286 +MT256917 +MT259227 +MT259238 +MT258382 +MT259246 +MT259253 +MT259254 +MT259255 +MT259259 +MT259265 +MT259284 +MT259252 +MT259229 +MT259230 +MT259260 +MT259285 +MT259278 +MT259280 +MT259247 +MT259240 +MT259243 +MT259249 +MT259250 +MT259251 +MT259256 +MT259258 +MT259266 +MT259267 +MT259287 +MT259241 +MT259242 +MT259228 +MT259236 +MT258383 +MT259248 +MT256918 +MT258380 +MT259235 +MT259237 +MT259239 +MT256924 +LC534419 +LC534418 +MT253704 +MT253710 +MT253701 +MT253702 +MT253703 +MT253706 +MT253707 +MT251972 +MT251973 +MT251975 +MT251976 +MT251978 +MT251979 +MT251977 +MT251980 +MT253696 +MT253697 +MT253698 +MT253699 +MT251974 +MT253700 +MT253705 +MT253709 +MT253708 +MT233526 +MT246667 +MT246451 +MT246453 +MT246454 +MT246461 +MT246462 +MT246490 +MT246450 +MT246452 +MT246464 +MT246470 +MT246474 +MT246480 +MT246481 +MT246482 +MT246457 +MT246459 +MT246466 +MT246489 +MT246456 +MT246458 +MT246475 +MT246476 +MT246477 +MT246479 +MT246487 +MT246449 +MT246455 +MT246468 +MT246469 +MT246486 +MT246488 +MT246467 +MT246478 +MT246485 +MT246460 +MT246463 +MT246465 +MT246471 +MT246472 +MT246473 +MT246483 +MT246484 +MT240479 +MT232869 +MT232870 +MT233522 +MT232871 +MT232872 +MT233520 +MT233523 +MT233519 +MT233521 +MT226610 +MT198653 +MT198651 +MT198652 +MT192759 +MT192765 +MT192772 +MT192773 +MT192758 +MT188341 +MT188339 +MT188340 +MT186680 +MT186676 +MT186677 +MT186679 +MT186678 +MT187977 +MT186681 +MT186682 +MT184912 +MT184910 +MT184911 +MT184913 +MT184909 +MT184907 +MT184908 +CADDYA000000000 +MT163712 +MT163716 +MT163719 +MT163720 +MT163715 +MT163721 +MT163714 +MT163717 +MT163737 +MT163738 +MT163718 +MT159706 +MT159707 +MT159717 +MT159716 +MT159719 +MT159709 +MT159710 +MT159712 +MT159713 +MT159714 +MT159722 +MT159711 +MT159715 +MT159718 +MT159720 +MT159721 +MT159708 +MT121215 +MT159778 +MT159705 +MT161607 +MT066156 +LC529905 +MT050493 +MT012098 +MT152824 +MT152900 +MT135043 +MT135042 +MT135041 +MT135044 +MT127116 +MT127113 +MT127114 +MT127115 +MT126808 +LC528233 +LC528232 +MT123290 +MT123291 +MT123292 +MT123293 +MT118835 +MT111896 +MT111895 +MT106052 +MT106053 +MT106054 +MT093571 +MT093631 +MT081059 +MT081068 +MT081060 +MT081061 +MT081065 +MT081067 +MT081062 +MT081063 +MT081064 +MT081066 +MT072667 +MT072688 +MT072668 +MT066158 +MT066157 +MT066159 +MT066175 +MT066176 +LC523807 +LC523808 +LC523809 +MT044258 +MT044257 +MT042777 +MT042778 +MT042776 +MT049951 +MT050414 +MT050415 +MT050417 +MT050416 +MT042774 +MT042775 +MT042773 +MT039887 +MT039888 +MT039890 +MT039873 +LC522350 +MT027062 +MT027063 +MT027064 +MT020781 +MT019530 +MT019531 +MT020881 +MT019533 +MT019529 +MT019532 +MT020880 +LR757995 +LR757996 +LR757997 +LR757998 +MT007544 +MT008023 +MT008022 +MN996530 +MN996531 +MN996527 +MN996528 +MN996529 +MN997409 +MN994468 +MN988668 +MN988669 +MN994467 +MN988713 +MN938387 +MN938389 +MN975263 +MN975268 +MN975267 +MN938388 +MN938390 +MN975264 +MN975265 +MN975266 +MN938386 +MN938385 +MN938384 +MN975262 +MN985325 +MN970003 +MN970004 +NC_045512 +MN908947 -- cgit v1.2.3