斯坦福在线人工智能课程字幕下载脚本
2011年10月12日 20:42
updated at 2011.11.02: 根据留言问题更新了一下脚本
updated at 2011.12.13: youtube改版, 改一下第一个脚本中某个正则
提供一种思路哈,方便离线看的童鞋~
批量下载youtube视频可以用firefox的插件BYTubeD~
字幕批量下载的步骤(firefox中):
1.选中youtube网页中那一堆视频的缩略图,右键选"查看选中部分源代码",存为in.txt,用下面的脚本正则出视频地址和名字:
# coding: utf-8
import sys,re
urlpattern =re.compile(r'watch\?v=([^\"]*)\" class=\"ux-thumb')
#titlepattern=re.compile(r'title=\"([^\"]*)\"')
titlepattern=re.compile(r'video-title\">([^\<]*)\<')
timepattern =re.compile(r'video-time\">(\d+:\d+)')
# in.txt is a part of the source file of the youtube page !!!
content=file('in.txt').read().split('\n')
allurl=[]
alltitle=[]
alltime=[]
for line in content:
allurl+=urlpattern.findall(line)
alltitle+=titlepattern.findall(line)
alltime+=timepattern.findall(line)
print "Found:",len(alltime),len(allurl),len(alltitle)
fo=file('urlinfo.txt','w')
sys.stdout=fo
allinfo=zip(allurl,alltitle,alltime)
for info in allinfo:
for i in info:
print i
print ''
fo.close()
2.对于上面的脚本生成的urlinfo.txt文件,用下一个脚本下载所有的xml格式的英文字幕,其中用到了http代理:
import os,urllib2
proxy_handler = urllib2.ProxyHandler({"http" : 'http://127.0.0.1:8123'})
opener = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener)
#baseurl='http://www.youtube.com/api/timedtext?v=%s&name=English%%20via%%20dotsub&lang=en&hl=en'
baseurl='http://www.youtube.com/api/timedtext?asr_langs=en%%2Cja&caps=asr&v=%s&name=English%%20via%%20dotsub&lang=en'
urlinfo=file('urlinfo.txt','r').read().split('\n')
cnt=len(urlinfo)
print 'Total:',cnt/4
for i in range(0,cnt-1,4):
try:
print 'Start',i,urlinfo[i+1]
if os.path.exists(urlinfo[i+1]+'.xml'):
continue
fo=file(urlinfo[i+1]+'.xml','w')
print 'getting:',baseurl % urlinfo[i]
fo.write(urllib2.urlopen(baseurl % urlinfo[i]).read())
fo.close()
print 'Done ',i
except Exception,e:
print e.message
pass
3.用下面的脚本把xml格式的字幕转换为srt格式的:
# coding: utf-8
import os,re
stamppattern=re.compile(r'start=\"(\d+)\.?\d*\" dur=\"(\d+)\.?\d*\"')
textpattern =re.compile(r'dur=\"\d+\.?\d*\">(.*)$')
def second2time(sec):
ret=[]
ret.append(sec/60/60)
ret.append(sec/60)
ret.append(sec%60)
return ret
def xml2srt(data,fo):
data=data.split('</text>')
data=data[:-1]
for i,line in enumerate(data):
fo.write('%d\n' % (i+1))
stamps=map(int,stamppattern.findall(line)[0])
fo.write('%02d:%02d:%02d,000 --> %02d:%02d:%02d,000\n' % tuple(second2time(stamps[0])+second2time(stamps[0]+stamps[1])))
fo.write(textpattern.findall(line)[0]+'\n\n\n')
if __name__=='__main__':
fs=filter(lambda s:s.endswith('.xml'),os.listdir('.'))
for f in fs:
if os.path.exists(f[:-4]+'.srt'): continue
print f[:-4],'ON!'
fo=file(f[:-4]+'.srt','w')
xml2srt(file(f,'r').read(),fo)
fo.close()
print f[:-4],'Done'
4.最后一步把字幕里的html标记替换掉:
# coding: utf-8
import os,re
def all_mark():
markpattern=re.compile(r'&[^;]*;')
allmarks=set()
fs=filter(lambda s:s.endswith('.srt'),os.listdir('.'))
for f in fs:
marks=markpattern.findall(file(f,'r').read())
allmarks=allmarks.union(set(marks))
return allmarks
# set(['&quot;', '&#39;', '&gt;', '&lt;'])
if __name__=='__main__':
fs=filter(lambda s:s.endswith('.srt'),os.listdir('.'))
for f in fs:
content=file(f,'r').read()
content=content.replace('&quot;','"')
content=content.replace('&#39;',"'")
content=content.replace('&gt;','>')
content=content.replace('&lt;','<')
fo=file(f,'w')
fo.write(content)
fo.close()
print 'Unreplace marks:',str(all_mark())
5.调整字幕时间,把上一条字幕的结束时间设为下一条字幕的开始时间
# coding: utf-8
import os,re
if __name__=='__main__':
fs=filter(lambda s:s.endswith('.srt'),os.listdir('.'))
for f in fs:
content=file(f,'r').read().split('\n')
n=len(content)/5
for i in range(n-1):
nextst=content[i*5+6][0:12]
content[i*5+1]=content[i*5+1][0:17]+nextst
of=file(f,'w')
of.write('\n'.join(content))
of.close()
