When I am programming in Ubuntu with python2.7 and MySQLdb, I had an error when I use other languages in python. English only doesn't make this error.
Traceback (most recent call last):
File "crawl.py", line 242, in <module>
parseArticle( u )
File "crawl.py", line 146, in parseArticle
gatherNeighborInfo( soup )
File "crawl.py", line 69, in gatherNeighborInfo
db.updateURL( url , '자신의 글 주소들을 db에 저장합니다' )
File "crawl.py", line 211, in updateURL self.cursor.execute("UPDATE urls SET state=%d,content='%s' WHERE url='%s'"%(state,content,url))
UnicodeDecodeError: 'ascii' codec can't decode byte 0xec in position 33: ordinal not in range(128)
So i tried to change ascii to utf-8. I made a file named sitecustomize.py on the /usr/local/lib/python2.7/site-packages. and sitecustomize.py source code is below.
import sys
sys.setdefaultencoding("utf-8")
but there's nothing changed. please help me. here is the whole source code.
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import robotparser
import urllib2
import time, traceback, re
import MySQLdb
crawler_name = 'daum_blog_crawler'
mainpage = 'http://blog.daum.net/'
# robot parser setting.
rp = robotparser.RobotFileParser( mainpage + 'robots.txt' )
rp.read()
def canFetch( url ):
return rp.can_fetch( crawler_name, url )
def getContent( url, delay=1):
time.sleep( delay )
if not canFetch( url ):
#print 'This url can NOT be fetched by our crawler :', url
return None
try:
opener = urllib2.build_opener()
opener.addheaders = [('User-agent',crawler_name)]
contents = opener.open(url).read()
except:
traceback.print_exc()
return None
return contents
def getArticleInfo( soup ):
rBlog = re.compile('.+blog.daum.net/\w+/\d+.*?')
URLs = soup('a',{'href':rBlog})
return [ u.get('href').split('?')[0] for u in URLs ]
def getOwnArticles( contents ):
ret = []
soup = BeautifulSoup( contents )
rBlog = re.compile('.+/BlogTypeView.+')
for u in soup('a',{'href':rBlog}):
href = u.get('href')
article = href.split('articleno=')[1].split('&')[0]
if ret.count(article)<1:
ret.append( article )
return ret
def gatherNeighborInfo( soup ):
rBlog = re.compile('http://blog.daum.net/\w+')
Neighbors = soup('a',{'href':rBlog})
cnt = 0
for n in Neighbors:
url = n.get('href')
blogname = url.split('/')[-1]
if url and url.startswith('http://') and db.isCrawledURL(url)<1:
db.insertURL( url, 1 )
db.updateURL( url , '자신의 글 주소들을 db에 저장합니다' )
url2 = getRedirectedURL( url )
if not url2: continue
re_url = 'http://blog.daum.net' + url2
body = getContent( re_url, 0 )
if body:
for u in getOwnArticles( body ):
fullpath = 'http://blog.daum.net/'+blogname+'/'+u
cnt+=db.insertURL( fullpath )
if cnt>0: print '%d neighbor articles inserted'%cnt
def getRedirectedURL( url ):
contents = getContent( url )
if not contents: return None
#redirect
try:
soup = BeautifulSoup( contents )
frame = soup('frame')
src = frame[0].get('src')
except:
src = None
return src
def getBody( soup, parent ):
rSrc = re.compile('.+/ArticleContentsView.+')
iframe = soup('iframe',{'src':rSrc})
if len(iframe)>0:
src = iframe[0].get('src')
iframe_src = 'http://blog.daum.net'+src
req = urllib2.Request( iframe_src )
req.add_header('Referer', parent )
body = urllib2.urlopen(req).read()
soup = BeautifulSoup( body )
return str(soup.body)
else:
print 'NULL contents'
return ''
def parseArticle( url ):
article_id = url.split('/')[-1]
blog_id = url.split('/')[-2]
if blog_id.isdigit():
print 'digit:', url.split('/')
newURL = getRedirectedURL( url )
if newURL:
newURL = 'http://blog.daum.net'+newURL
print 'redirecting', newURL
contents = getContent( newURL, 0 )
if not contents:
print 'Null Contents...'
db.updateURL( url, -1 )
return
soup = BeautifulSoup( contents )
gatherNeighborInfo( soup )
n=0
for u in getArticleInfo( soup ):
n+=db.insertURL( u )
if n>0: print 'inserted %d urls from %s'%(n,url)
sp = contents.find('<title>')
if sp>-1:
ep = contents[sp+7:].find('</title>')
title = contents[sp+7:sp+ep+7]
else:
title = ''
contents = getBody( soup, newURL )
pStyle = re.compile('<style(.*?)>(.*?)</style>', re.IGNORECASE | re.MULTILINE | re.DOTALL )
contents = pStyle.sub('', contents)
pStyle = re.compile('<script(.*?)>(.*?)</script>', re.IGNORECASE | re.MULTILINE | re.DOTALL )
contents = pStyle.sub('', contents)
pStyle = re.compile("<(.*?)>", re.IGNORECASE | re.MULTILINE | re.DOTALL )
contents = pStyle.sub("", contents)
db.updateURL( url , '처리했다고 db에 표시합니다.' )
else:
print 'Invalid blog article...'
db.updateURL( url, 'None', -1 )
class DB:
"MySQL wrapper class"
def __init__(self):
self.conn = MySQLdb.connect(db='crawlDB', user='root', passwd='......')
self.cursor = self.conn.cursor()
self.cursor.execute('CREATE TABLE IF NOT EXISTS urls(url CHAR(150), state INT, content TEXT)')
def commit(self):
self.conn.commit()
def __del__(self):
self.conn.commit()
self.cursor.close()
def insertURL(self, url, state=0, content=None):
if url[-1]=='/': url=url[:-1]
try:
self.cursor.execute("INSERT INTO urls VALUES ('%s',%d,'%s')"%(url,state,content))
except:
return 0
else:
return 1
def selectUncrawledURL(self):
self.cursor.execute('SELECT * FROM urls where state=0')
return [ row[0] for row in self.cursor.fetchall() ]
def updateURL(self, url, content, state=1):
if url[-1]=='/': url=url[:-1]
self.cursor.execute("UPDATE urls SET state=%d,content='%s' WHERE url='%s'"%(state,content,url))
def isCrawledURL(self, url):
if url[-1]=='/': url=url[:-1]
self.cursor.execute("SELECT COUNT(*) FROM urls WHERE url='%s' AND state=1"%url)
ret = self.cursor.fetchone()
return ret[0]
db = DB()
if __name__=='__main__':
print 'starting crawl.py...'
contents = getContent( mainpage )
URLs = getArticleInfo( BeautifulSoup( contents ) )
nSuccess = 0
for u in URLs:
nSuccess += db.insertURL( u )
print 'inserted %d new pages.'%nSuccess
while 1:
uncrawled_urls = db.selectUncrawledURL()
if not uncrawled_urls: break
for u in uncrawled_urls:
print 'downloading %s'%u
try:
parseArticle( u )
except:
traceback.print_exc()
db.updateURL( u, -1 )
db.commit()
#bs.UpdateIndex()
