import sys
import urlparse
import os
+from cgi import parse_header
class InvalidICS(Exception): pass
content = sys.stdin.read()
return (content, encoding)
- if not parsedURL[0]:
- try: content = open(os.path.abspath(url),'r').read()
- except (IOError, OSError), e:
- sys.stderr.write('%s\n'%e)
- sys.exit(1)
- return (content, encoding)
+ if not parsedURL[0]: url = 'file://' + os.path.abspath(url)
# If we've survived, use python's generic URL opening library to handle it
import urllib2
try:
res = urllib2.urlopen(url)
content = res.read()
+ ct = res.info().getplist()
res.close()
except (urllib2.URLError, OSError), e:
sys.stderr.write('%s\n'%e)
sys.exit(1)
+
+ for param in ct:
+ if 'charset' in param:
+ encoding = param.split('=')[1]
+ break
+
return (content, encoding)
except ImportError:
import urllib2
- if not url: return ''
+ if not url: return ('','')
- encoding = '' # If we don't populate this, the script will assume UTF-8
+ if not 'http' in urlparse.urlparse(url)[0]: return ('','')
if 'httplib2' in sys.modules:
try: h = httplib2.Http('.httplib2-cache')
except OSError: h = httplib2.Http()
else: h = False
- try:
- if h:
+ if h:
+ try:
req = h.request(url)
- content = req[1]
- if 'content-type' in req[0]:
- for ct in req[0]['content-type'].split(';'):
- ct = ct.lower()
- if 'charset' in ct:
- encoding = ct.split('=')[1].strip()
- return (content, encoding)
- except ValueError, e:
- sys.stderr.write('%s\n'%e)
- sys.exit(1)
+ except ValueError, e:
+ sys.stderr.write('%s\n'%e)
+ sys.exit(1)
- try:
- content = urllib2.urlopen(url).read()
- return (content, encoding)
- except (urllib2.URLError, OSError), e:
- sys.stderr.write('%s\n'%e)
- sys.exit(1)
+ resp, content = req
+ if 'content-type' in resp:
+ ct = 'Content-Type: %s'%req[0]['content-type']
+ ct = parse_header(ct)
+ if 'charset' in ct[1]: encoding = ct[1]['charset']
+ else: encoding = ''
+ else:
+ ct = ''
+ encoding = ''
- return ('', '')
+ else:
+ try:
+ req = urllib2.urlopen(url)
+ except urllib2.URLError, e:
+ sys.stderr.write('%s\n'%e)
+ sys.exit(1)
+
+ content = req.read()
+ ct = req.info().getplist()
+ for param in ct:
+ if 'charset' in param:
+ encoding = param.split('=')[1]
+ break
+
+ return (content, encoding)
def generateRules():