-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #5 from lrm22005/Luis
Luis updating codes
- Loading branch information
Showing
7 changed files
with
109 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,101 @@ | ||
import requests | ||
from Bio import Entrez | ||
from io import BytesIO | ||
import time | ||
import os | ||
import logging | ||
import json | ||
|
||
class GenePubMedDownloader: | ||
def __init__(self, api_key, email, output_dir, max_records_per_query=9999, checkpoint_file="checkpoint.json"): | ||
self.base_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/" | ||
self.api_key = api_key | ||
self.email = email | ||
Entrez.email = email # Set email for NCBI E-utilities | ||
self.max_records_per_query = max_records_per_query | ||
self.output_dir = output_dir | ||
os.makedirs(self.output_dir, exist_ok=True) | ||
self.checkpoint_file = checkpoint_file | ||
self.checkpoint_data = self.load_checkpoint() | ||
|
||
def fetch_pubmed_data(self, gene_name): | ||
if gene_name in self.checkpoint_data: | ||
logging.info(f"Skipping {gene_name} (already processed)") | ||
return self.checkpoint_data[gene_name] | ||
normalized_gene = gene_name.replace(" ", "_").replace("/", "_").replace("\\", "_") | ||
records = [] | ||
attempt = 0 | ||
max_attempts = 5 | ||
|
||
while attempt < max_attempts: | ||
try: | ||
search_url = f"{self.base_url}esearch.fcgi?db=pubmed&term={gene_name}[Gene Name]&retmax=1&api_key={self.api_key}&usehistory=y" | ||
search_response = requests.get(search_url, timeout=10) | ||
if search_response.status_code == 200: | ||
search_results = Entrez.read(BytesIO(search_response.content)) | ||
if 'WebEnv' in search_results and 'QueryKey' in search_results: | ||
webenv = search_results['WebEnv'] | ||
query_key = search_results['QueryKey'] | ||
count = int(search_results['Count']) | ||
logging.info(f"Total records found for {gene_name}: {count}") | ||
if count > 0: | ||
for start in range(0, count, self.max_records_per_query): | ||
fetch_url = f"{self.base_url}efetch.fcgi?db=pubmed&rettype=medline&retmode=text&retstart={start}&retmax={self.max_records_per_query}&webenv={webenv}&query_key={query_key}&api_key={self.api_key}" | ||
fetch_response = requests.get(fetch_url, timeout=10) | ||
records.append(fetch_response.text) | ||
logging.info(f"Fetched records for {gene_name} starting from {start}") | ||
else: | ||
logging.info(f"No records found for {gene_name}.") | ||
return None | ||
else: | ||
logging.error(f"No WebEnv/QueryKey found in the search results for {gene_name}.") | ||
return None | ||
file_path = self.save_records_to_file(normalized_gene, records) | ||
self.checkpoint_data[gene_name] = file_path | ||
self.save_checkpoint() | ||
return file_path | ||
break | ||
except requests.exceptions.RequestException as e: | ||
attempt += 1 | ||
logging.error(f"Attempt {attempt}: An error occurred: {e}") | ||
time.sleep(2 ** attempt) | ||
return None | ||
|
||
def save_records_to_file(self, gene_name, records): | ||
filename = f"{gene_name}.txt" | ||
file_path = os.path.join(self.output_dir, filename) | ||
with open(file_path, 'w', encoding='utf-8') as file: | ||
file.write("\n".join(records)) | ||
logging.info(f"Saved records for {gene_name} to {file_path}") | ||
return file_path | ||
|
||
def load_checkpoint(self): | ||
if os.path.exists(self.checkpoint_file): | ||
with open(self.checkpoint_file, 'r') as file: | ||
return json.load(file) | ||
return {} | ||
|
||
def save_checkpoint(self): | ||
with open(self.checkpoint_file, 'w') as file: | ||
json.dump(self.checkpoint_data, file) | ||
|
||
def load_gene_names(file_path): | ||
with open(file_path, 'r', encoding='utf-8') as file: | ||
return [line.strip() for line in file if line.strip()] | ||
|
||
# Setup logging | ||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') | ||
|
||
# Example Usage | ||
api_key = "19bea34a4dbdbc6ef30392cee15943365309" | ||
email = "lrmercadod@gmail.com" | ||
output_dir = "./gene_based_records/" | ||
downloader = GenePubMedDownloader(api_key, email, output_dir) | ||
|
||
# Load gene names and symbols | ||
full_names = load_gene_names('./data/gene_name_info/query_full_name.txt') | ||
symbols = load_gene_names('./data/gene_name_info/query_symbol.txt') | ||
|
||
# Fetch records for each gene name and symbol | ||
for gene in full_names + symbols: | ||
downloader.fetch_pubmed_data(gene) |
File renamed without changes.