-
Notifications
You must be signed in to change notification settings - Fork 4
Expand file tree
/
Copy pathurl.py
More file actions
64 lines (53 loc) · 2.31 KB
/
Copy pathurl.py
File metadata and controls
64 lines (53 loc) · 2.31 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#import urllib2
from six.moves.urllib.request import urlopen,Request
from six.moves.urllib.error import HTTPError
import gzip
import io
import logging
import json
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
class URLResource:
def __init__(self, url):
self._url = url
self._lines = None
def get_lines(self):
if self._lines is not None:
return self._lines
else:
try:
request = Request(self._url)
request.add_header('User-Agent', 'Mozilla/5.0')
response = urlopen(request)
except HTTPError as error:
logger.error(str(error) + "\n\t " + self._url)
return []
raw_content = response.read()
content = raw_content
response_info = response.info()
if response_info.get('Content-Type') == 'application/json' :
json_obj = json.loads(content.decode('utf-8'))
lines = json_obj
else:
if response_info.get('Content-Type') == 'application/x-gzip' or \
response_info.get('Content-Encoding') == 'gzip' or \
self._url.endswith('.gz'):
content = gzip.GzipFile(fileobj=io.BytesIO(raw_content)).read()
content_decoded = content.decode('utf-8')
#to get lines, split by \n
lines = content_decoded.split("\n")
self._lines = lines
return self._lines
if __name__ == '__main__':
url_resource = URLResource('http://geneontology.org/ontology/go.obo')
lines = url_resource.get_lines()
print("%s\n\t%d lines" % (url_resource._url, len(lines)))
url_resource = URLResource('ftp://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/Homo_sapiens.gene_info.gz')
lines = url_resource.get_lines()
print("%s\n\t%d lines" % (url_resource._url, len(lines)))
url_resource = URLResource('http://www.geneontology.org/gene-associations/goa_human.gaf.json')
lines = url_resource.get_lines()
print("%s\n\t%s" % (url_resource._url, lines))
url_resource = URLResource('http://www.brenda-enzymes.info/ontology/tissue/tree/update/update_files/BrendaTissueOBO')
lines = url_resource.get_lines()
print("%s\n\t%s" % (url_resource._url, lines))