content = sys.stdin.read()
return (content, encoding)
- if not parsedURL[0]:
- try: content = open(os.path.abspath(url),'r').read()
- except (IOError, OSError), e:
- sys.stderr.write('%s\n'%e)
- sys.exit(1)
- return (content, encoding)
+ if not parsedURL[0]: url = 'file://' + os.path.abspath(url)
# If we've survived, use python's generic URL opening library to handle it
import urllib2
try:
res = urllib2.urlopen(url)
content = res.read()
+ if 'content-type' in res.info(): ct = res.info()['content-type']
+ else: ct = ''
res.close()
except (urllib2.URLError, OSError), e:
sys.stderr.write('%s\n'%e)
sys.exit(1)
+
+ encoding = 'charset' in ct and ct.split(';')[-1].lower().split('=')[-1].strip() or ''
return (content, encoding)
except ImportError:
import urllib2
- if not url: return ''
-
- encoding = '' # If we don't populate this, the script will assume UTF-8
+ if not url: return ('','')
+ if not 'http' in parsedURL[0]: return ('','')
if 'httplib2' in sys.modules:
try: h = httplib2.Http('.httplib2-cache')
except OSError: h = httplib2.Http()
else: h = False
- try:
- if h:
+ if h:
+ try:
req = h.request(url)
- content = req[1]
- if 'content-type' in req[0]:
- for ct in req[0]['content-type'].split(';'):
- ct = ct.lower()
- if 'charset' in ct:
- encoding = ct.split('=')[1].strip()
- return (content, encoding)
- except ValueError, e:
- sys.stderr.write('%s\n'%e)
- sys.exit(1)
+ except ValueError, e:
+ sys.stderr.write('%s\n'%e)
+ sys.exit(1)
- try:
- content = urllib2.urlopen(url).read()
- return (content, encoding)
- except (urllib2.URLError, OSError), e:
- sys.stderr.write('%s\n'%e)
- sys.exit(1)
+ content = req[1]
+ if 'content-type' in req[0]: ct = req[0]['content-type']
+
+ else:
+ try:
+ req = urllib2.urlopen(url)
+ except urllib2.URLError, e:
+ sys.stderr.write('%s\n'%e)
+ sys.exit(1)
+
+ content = req.read()
+ info = req.info()
- return ('', '')
+ ct = info['content-type']
+
+ encoding = 'charset' in ct and ct.split(';')[-1].lower().split('=')[-1].strip() or ''
+
+ return (content, encoding)
def generateRules():