Uzanto:Maksim/aald
- ivsen.py
# -*- coding: UTF-8 -*-
__version__='$Id:'
import wikipedia, pagegenerators
import re, sys, os
import codecs
import perevod,imagetransfer1
from bib_tekst import *
from get_buf import *
from bib_kateg import *
te_bildoj = {
'ru':u'изображений',
'eo':u'bildoj',
}
te_ekslj= {
'ru':u'внешних ссылок',
'eo':u'eksteraj ligiloj',
}
te_komunejoj = {
'ru':u'викискладов',
'eo':u'komunejoj',
}
te_ligoj = {
'ru':u'ссылок',
'eo':u'ligoj',
}
te_tabeloj = {
'ru':u'таблиц',
'eo':u'tabeloj',
}
te_kategorioj = {
'ru':u'категорий',
'eo':u'kategorioj',
}
te_vi_di = {
'ru':u'см. обсуждение',
'eo':u'vidu diskuton',
}
te_diskuto= {
'ru':u'обсуждение',
'eo':u'diskuto',
}
te_al_bildoj = {
'ru':u'Дополнительные изображения',
'eo':u'Aldonaj bildoj',
}
te_al_ekslj= {
'ru':u'Дополнительные внешние ссылки',
'eo':u'Aldonaj eksteraj ligiloj',
}
te_aldon= {
'ru':u'Добавления',
'eo':u'Aldonaĵoj',
}
te_aldon_kat= {
'ru':u'Категория:Добавления для статей',
'eo':u'Kategorio:Aldonajxoj por artikoloj',
}
def vivod(b):
wikipedia.output(b)
otch.write(b)
otch.flush()
def ubrkoment (text):
# Ignore tekst within nowiki tags and HTML comments
nowikiOrHtmlCommentR = re.compile(r'<nowiki>.*?|<!--.*?-->', re.IGNORECASE | re.DOTALL)
match = nowikiOrHtmlCommentR.search(text)
while match:
text = text[:match.start()] + text[match.end():]
match = nowikiOrHtmlCommentR.search(text)
text=text.replace(u'-->',u'-- >')
return text
def prov_v_koment (text,pm):
# Ignore tekst within nowiki tags and HTML comments
nowikiOrHtmlCommentR = re.compile(r'.*?|<!--.*?-->', re.IGNORECASE | re.DOTALL)
for match in nowikiOrHtmlCommentR.finditer(text):
for p in pm:
if p>=match.start() and p<match.end():
return 1
return 0
def provcifr(t):
i=ord(t[0])
return (i>=ord(u'0')) & (i<=ord(u'9'))
def prov_te_format_kart(t):
minrzm=50
t=ubr_nk_prob(t)
if (t==u'thumb') or (t==u'thumbnail') or (t==u'frame'):
return 1
if (t==u'left') or (t==u'center') or (t==u'right'):
return 1
if len(t)<3:
return 0
x=0
i=0
while i<6 and i<len(t) and provcifr(t[i]):
x=x*10+ord(t[i])-ord(u'0')
i+=1
if (i>=1) and (t[i:]==u'px'):
if x<minrzm:
return 2
return 1
return 0
def linkedPages_podp(site,thistxt):
"""Gives the normal (not-interwiki, non-category) pages the page
links to, as a list of Page objects
"""
result = []
result_podp = []
thistxt = wikipedia.removeLanguageLinks(thistxt)
thistxt = wikipedia.removeCategoryLinks(thistxt, site)
thistxt = ubrkoment (thistxt)
# [[(?P<title>[^]|]*)(|[^]]*)?]]
i=0
Rlink = re.compile(r'\[\[(?P<title>[^\]\|]*)(\|[^\]]*)?\]\]')
for match in Rlink.finditer(thistxt):
title = match.group('title')
if site.lang == 'eo':
title = wikipedia.resolveEsperantoXConvention(title)
page = wikipedia.Page(site, title)
podp=u''
j=iskat(thistxt[match.start():match.end()],u'|')
if j!=-1:
j+=match.start()+1
kon=iskkonpodp(thistxt[j:],0,u'[[',u']]')
if kon!=-1:
podp=thistxt[j:j+kon]
result.append(page)
result_podp.append(podp)
i+=1
return (result,result_podp)
def razb_arg(t):
r=[]
p0=0
while p0<len(t):
p1=iskats(t,p0,u'|')
if p1==-1:
p1=len(t)
a=t[p0:p1]
a=ubr_nk_prob(a)
r.append(a)
p0=p1+1
return r
def image_podp(site,thistxt):
"""
Gives the images the page shows, as a list of Page objects.
This includes images in galleries.
"""
result = []
# Find normal images
(res1,res1_podp)=linkedPages_podp(site,thistxt)
j=0
while j<len(res1):
if res1[j].isImage():
result.append(imagetransfer1.Izobr(res1[j].title(),site.lang,res1_podp[j]))
j+=1
# Find images in galleries
galleryR = re.compile('<gallery>.*?</gallery>', re.DOTALL)
galleryEntryR = re.compile('(?P<title>(%s|%s):.+?)(\|.+)?\n' % (site.image_namespace(), site.family.image_namespace(code = '_default')))
for gallery in galleryR.findall(thistxt):
# for match in galleryEntryR.finditer(gallery):
# page = wikipedia.Page(site, match.group('title'))
# result.append(imagetransfer1.Izobr(page.title(),site.lang,u''))
for match in gallery.split(u'\n'):
# vivod(u'match=%s\n'%match)
podp=u''
p=iskat(match,u'|')
if p!=-1:
podp=ubr_nk_prob(match[p+1:])
else:
p=len(match)
ti=ubr_nk_prob(match[:p])
# vivod(u'ti=%s\n'%ti)
if len(ti)<1:
continue
page = wikipedia.Page(site, ti)
if not page.isImage():
continue
result.append(imagetransfer1.Izobr(page.title(),site.lang,podp))
j=0
while j<len(result):
t=result[j].iz
p=iskat(t,u':')
if p!=-1:
result[j].iz=t[p+1:]
t=result[j].podp
tn=u''
fpred=0
am=razb_arg(t)
for a in am:
ptf=prov_te_format_kart(a)
if ptf==0:
if fpred:
tn=tn+'|'
fpred=1
tn=tn+a
elif ptf==2:
result[j].tip=100
vivod(u' - m px %s\n'%result[j].iz)
break
result[j].podp=tn
j+=1
return result
class Ssilk:
def __init__(self, ss,jaz,podp):
self.ss = ss
self.jaz = jaz
self.podp = podp
def sravn(self,e1):
return self.ss!=e1.ss
# def tekst(self):
# return u'[[%s]]'%self.ss
def isk_ssilk(site,thistxt):
result = []
(res1,res1_podp)=linkedPages_podp(site,thistxt)
j=0
while j<len(res1):
if not res1[j].isImage():
result.append(Ssilk(res1[j].title(),site.lang,res1_podp[j]))
j+=1
return result
def isk_ssilk_smt(site,thistxt):
result = []
st=thistxt.split(u'\n')
for t in st:
p=iskats(t,0,u'[[')
if p==-1 or p>4:
continue
(res1,res1_podp)=linkedPages_podp(site,t)
j=0
# while j<len(res1):
if len(res1)>=1:
if not res1[j].isImage():
result.append(Ssilk(res1[j].title(),site.lang,res1_podp[j]))
j+=1
return result
def perv_upper(t):
if len(t)<1:
return u''
return t[0].upper()+t[1:]
class Shabl:
def __init__(self, nazv,jaz,arg):
self.nazv=perv_upper(nazv)
self.jaz=jaz
self.arg=[]
self.neisp=0
for a in arg:
self.arg.append(a)
def sravn(self,e1):
if len(self.arg)!=len(e1.arg):
return 1
if self.nazv!=e1.nazv:
return 1
for i in range(len(self.arg)):
if self.arg[i]!=e1.arg[i]:
return 1
return 0
def sravn_com(self,e1):
if len(self.arg)<1 or len(e1.arg)<1:
return 1
if self.nazv!=e1.nazv:
return 1
if (perv_upper(self.arg[0]).replace(u'_',u' ')!=
perv_upper(e1.arg[0]).replace(u'_',u' ')):
return 1
return 0
def tekst(self):
t=u'{{'+self.nazv
for a in self.arg:
t+=u'|'+a
t+=u'}}'
return t
def tekstn(self,n):
t=u'{{'+self.nazv
i=0
while i<n and i<len(self.arg):
a=self.arg[i]
t+=u'|'+a
i+=1
t+=u'}}'
return t
def isk_shabl(jaz,t,z):
r=[]
zm=[]
for z1 in z:
zm.append(z1[0].lower()+z1[1:])
p0=0
while p0<len(t):
p=iskats(t,p0,u'{{')
if p==-1:
break
pn=p+2
# if pn>=len(t)-1:
# break
# if t[pn]==u' ' or t[pn]==u'\n':
# pn+=1
# if t[pn:].startswith(z)) or t[pn:].startswith(z):
pk=iskkonpodp(t,pn,u'{{',u'}}')
if pk==-1:
break
am=razb_arg(t[pn:pk])
if len(am)>=1:
fsovp=0
for z1 in z:
if am[0]==z1:
fsovp=1
for z1 in zm:
if am[0]==z1:
fsovp=1
if fsovp:
am1=[]
for a in am[1:]:
am1.append(a)
r.append(Shabl(am[0],jaz,am1))
p0=pk+2
return r
def korr_com_shabl(sh):
if len(sh.arg)<1:
sh.neisp=1
# vivod(u' <-shcom len(arg)=0\n')
return
if sh.nazv==u'Commonscat':
sh.nazv=u'Commons'
sh.arg[0]=u'Category:'+sh.arg[0]
if sh.nazv!=u'Commons':
return
if sh.arg[0].startswith(u':Category:'):
sh.arg[0]=sh.arg[0][1:]
if sh.arg[0].startswith(u'Category:'):
sh.arg[0]=sh.arg[0][:9]+perv_upper(sh.arg[0][9:])
sh.arg[0]=sh.arg[0].replace(u'_',u' ')
class Eksl:
def __init__(self, adr,jaz,podp):
p=iskats(adr,8,'/')
if p==-1:
p=len(adr)
adr=adr[:p].lower()+adr[p:]
self.adr = adr
self.jaz = jaz
self.podp = podp
self.podp2 = u''
self.tip=100 # >=80 =nelzja ispolz
self.tip2=100
self.prioritet=0
self.njazisp=1
self.njazisp2=0
def vz_prioritet(self):
return self.prioritet
def sravn(self,e1):
return self.adr!=e1.adr
def tekst(self):
return u'%s %s'%(self.adr,self.podp)
def pertekst(self):
return u'%s %s'%(self.adr,self.perpodp.t)
def isk_eksl(jaz,t):
r=[]
p0=0
while p0<len(t):
p=iskats(t,p0,u'http://')
if p==-1:
break
flks=((p>0 and t[p-1]==u'[') or (p>1 and t[p-2:p]==u'[ '))
if flks:
pk=iskkonpodp(t,p,u'[',u']')
if pk==-1:
flks=0
if not flks:
pk=iskats(t,p,u'\n')
if pk==-1:
pk=len(t)
(p1,np1)=iskats_mn(t[:pk],p,[u' ',u'}',u')',u'<',u'>'])
if p1==-1:
p1=pk
adr=t[p:p1]
podp=t[p1+1:pk]
r.append(Eksl(adr,jaz,podp))
p0=p1
shm=isk_shabl(jaz,t,(u'El2',))
for sh in shm:
if len(sh.arg)>=1:
podp=u''
if len(sh.arg)>=2:
podp=sh.arg[1]
r.append(Eksl(u'http://'+sh.arg[0],jaz,podp))
for el in r:
while len(el.adr)>=1 and el.adr[len(el.adr)-1]==u'/':
el.adr=el.adr[:len(el.adr)-1]
return r
class Tabl:
def __init__(self, jaz,te):
self.jaz = jaz
self.te = te
def sravn(self,e1):
return self.te!=e1.te
def tekst(self):
return self.te+u'\n'
def isk_tabl(jaz,t):
ost=[u'{|',u'<table']
zst=[u'|}',u'</table>']
r=[]
p0=0
while p0<len(t):
(p,n)=iskats_mn(t,p0,ost)
if p==-1:
break
os=ost[n]
zs=zst[n]
pk=iskkonpodp(t,p+len(os),os,zs)
if pk==-1:
break
pk+=len(zs)
r.append(Tabl(jaz,t[p:pk]))
p0=pk
return r
t_zag_eksl = {
u'eo': (u'Eksteraj ligiloj', u'Ekstera ligilo', u'Eksteraj ligoj',
u'Ekstera ligo', u'Rete'),
u'io': (u'Externa ligili', u'Extera ligili'),
u'ru': (u'Внешние ссылки', u'Внешняя ссылка'),
}
t_zag_vian = {
u'de': (u'Siehe auch',u'Weitere themen'),
u'en': (u'See also',u'Miscellaneous topics'),
u'eo': (u'Vidu ankaux jenon:',u'Vidu ankaux jenon',u'Vidu ankaux'),
u'fr': (u'Voir aussi',),
u'io': (u'Videz anke',),
u'nl': (u'Zie ook',),
u'ru': (u'См. также', u'Смотри также', u'Смотрите также'),
}
def gl_zagolov(jaz,zag):
if not zag.has_key(jaz):
return u'*'
return zag[jaz][0]
def isk_zagolov(t,jaz,zag):
if not zag.has_key(jaz):
return (len(t),len(t))
tm=t.lower()
zj=zag[jaz]
p1=len(t)
while p1>1:
# if t[p1-1]==u'\n':
# p1-=1
p0=iskato(t,p1-2,u'\n')+1
for z in zj:
zm=z.lower()
i=iskats(tm[:p1],p0,zm)
if i!=-1:
fll=0
for c in t[p0:i]:
if c.isalnum():
fll=1
for c in t[i+len(z):p1]:
if c.isalnum():
fll=1
if prov_v_koment(t,(i,i+len(z),p1)):
fll=1
if fll==0:
return (p0,p1)
p1=p0
return (len(t),len(t))
def isk_kon_zagolov(t,p0,p1):
i=0
while p0+i<len(t) and t[p0+i]==u'=':
i+=1
if i==0:
i=100
if p1>0 and t[p1-1]==u'\n':
p1-=1
p=p1
while 1:
p=iskats(t,p,u'\n=')
if p==-1:
return len(t)
p+=1
j=1
while p+j<len(t) and t[p+j]==u'=':
j+=1
if j<=i:
return p
class Perev_podp:
def __init__(self,t):
self.t = t
self.ps = perevod.Perev_stat()
# self.ps = ps
def objed_podp(podp,podp1):
# if podp==u'':
# podp=podp1
# elif podp1!=u'':
# podp+=u' '+podp1
if podp.ps.nup+podp.ps.nvap < podp1.ps.nup+podp1.ps.nvap:
podp=podp1
return podp
def perevod_ob(slov,vhjaz,vihjaz,podp):
if podp==u'' or podp==u' ':
# return u''
return Perev_podp(u'')
(podpperev,st_perev)=perevod.perevod_stat(slov,vhjaz,vihjaz,podp)
# if fl_perev:
# podp=u'%s:%s %s->%s:%s' % (vhjaz,podp,vhjaz,vihjaz,podpperev)
# else:
# podp=u'%s:%s' % (vhjaz,podp)
# return podp
r=Perev_podp(podpperev)
r.ps=st_perev
return r
def prov_tekst_uzxe(t,s):
#X!!!
return iskats(t,0,s)!=-1
def prov_dobav_katcom1(nscom,zapriz,katcomsp,n):
n=perv_upper(ubr_nk_prob(n))
vivod(u' prov_dobav_katcom1 %s\n'%n)
p=iskats(n,0,u'#')
if p!=-1:
n=n[:p]
vivod(u' prov_dobav_katcom1# %s\n'%n)
t=u'Category:'+n
if nscom.has_key(t) and (not zapriz.has_key(n)):
t1=u'[['+t+u']]\n'
if not t1 in katcomsp:
katcomsp.append(t1)
vivod(u' <-- ++\n')
return 1
return 0
def prov_dobav_katcom(nscom,zapriz,katcomsp,vhs,jaz):
prov_dobav_katcom1(nscom,zapriz,katcomsp,vhs)
if jaz==u'en':
vhok=vhs[len(vhs)-1]
if vhok==u'y':
vhs1=vhs[:len(vhs)-1]+u'ies'
prov_dobav_katcom1(nscom,zapriz,katcomsp,vhs1)
else:
vhs1=vhs+u's'
prov_dobav_katcom1(nscom,zapriz,katcomsp,vhs1)
vhs1=vhs+u'es'
prov_dobav_katcom1(nscom,zapriz,katcomsp,vhs1)
def shcom_v_katcom(nscom,zapriz,katcomsp,sht):
vivod(u' shcom_v_katcom0 %d\n'%len(sht))
for sh in sht:
if sh.neisp or len(sh.arg)<1:
continue
n=sh.arg[0]
vivod(u' shcom_v_katcom0 %s\n'%n)
n2=n[iskat(n,u':')+1:]
usp=prov_dobav_katcom1(nscom,zapriz,katcomsp,n2)
if usp:
continue
sico=wikipedia.getSite(code = u'commons',fam = u'commons')
pli = wikipedia.Page(sico,n)
if pli.isCategory():
continue
try:
get1(pli)
ti = pli.get()
except wikipedia.NoPage:
b = u' - %s:%s\n' % (pli.site().lang,pli.title())
vivod(b)
continue
except wikipedia.IsRedirectPage:
n1=pli.getRedirectTarget()
b = u' + %s:%s\n -> %s\n' % (pli.site().lang,pli.title(),n1)
vivod(b)
n2=n1[iskat(n1,u':')+1:]
vivod(u' shcom_v_katcom %s\n'%n2)
usp=prov_dobav_katcom1(nscom,zapriz,katcomsp,n2)
if usp:
continue
pli = wikipedia.Page(pli.site(),pli.getRedirectTarget())
if pli.isCategory():
continue
try:
get1(pli)
ti = pli.get()
except wikipedia.NoPage:
b = u' - %s:%s\n' % (pli.site().lang,pli.title())
vivod(b)
continue
except wikipedia.IsRedirectPage:
n1=pli.getRedirectTarget()
n2=n1[iskat(n1,u':')+1:]
vivod(u' shcom_v_katcom %s\n'%n2)
prov_dobav_katcom1(nscom,zapriz,katcomsp,n2)
b = u' + %s:%s\n -> %s\n' % (pli.site().lang,pli.title(),pli.getRedirectTarget())
vivod(b)
continue
otch2.write(u'======shcom_v_katcom ti======\n%s\n-=======\n%s\n===============================\n' % (pli.title(),ti))
otch2.flush()
plikat = pli.categories()
vivod(u' shcom_v_katcom len plikat %d\n'%len(plikat))
for pka in plikat:
n1=pka.title()
vivod(u' shcom_v_katcom1 %s\n'%n1)
n2=n1[iskat(n1,u':')+1:]
prov_dobav_katcom1(nscom,zapriz,katcomsp,n2)
def shcom_prov_uzxe_povt(shcomi,shcomo):
for j in range(len(shcomo)):
sh=shcomo[j]
vivod(u' %s\n'%sh.tekst())
if sh.neisp:
vivod(u' <-neisp shcom\n')
continue
fuzxe=0
for sh0 in shcomi:
if not sh.sravn_com(sh0):
fuzxe=1
break
if fuzxe:
shcomo[j].neisp=1
vivod(u' <-uzxe shcom\n')
continue
fpovt=0
for k in range(j):
if (not shcomo[k].neisp) and (not sh.sravn_com(shcomo[k])):
fpovt=1
break
if fpovt:
shcomo[j].neisp=1
vivod(u' <-povt shcom\n')
continue
def put_prov(pl,nt,komm,me):
pl.put(nt, comment = komm, minorEdit = me)
return
ot=pl.get()
i=0
while i<3:
pl.put(nt, comment = komm, minorEdit = me)
pl1=wikipedia.Page(pl.site(), pl.title())
try:
t1=pl1.get()
except wikipedia.NoPage:
b = u' put_prov: - %s\n' % pl.title()
vivod(b)
t1=u''
except wikipedia.IsRedirectPage:
b = u' put_prov: - %s\n -> %s\n' % (pl.title(),pl.getRedirectTarget())
vivod(b)
return
if t1!='' and ot!=t1:
break
vivod(u'!!!! neud put() %s'%pl.title())
pl=pl1
i+=1
def vibor_iwi(tpl):
nmax=8
tpri={u'en': 0, u'de': 1, u'ja': 2, u'fr': 3, u'pl': 4, u'nl': 5, u'it': 6,
u'es': 7, u'sv': 8, u'pt': 9, u'zh': 10, u'he': 11, u'ru': 12,
u'no': 13, u'fi': 14, u'cs': 15, u'da': 16, u'hu': 17, u'eo': 18,
u'ca': 19, u'bg': 20, u'sl': 21, u'sk': 22, u'et': 23, u'sr': 24,
u'id': 25, u'ro': 26, u'nn': 27, u'ko': 28, u'gl': 29, u'hr': 30,
u'uk': 31, u'lt': 32, u'ms': 33, u'vi': 34, u'tr': 35, u'th': 36,
u'io': 37, u'el': 38, u'ar': 39, u'is': 40, u'lb': 41, u'tt': 42,
u'simple': 43, u'bs': 44, u'ka': 45, u'fa': 46, u'sh': 47, u'af': 48,
u'br': 49, u'la': 50,}
ts=[]
for i in range(len(tpl)):
jaz=tpl[i].site().lang
z=1000
if tpri.has_key(jaz):
z=tpri[jaz]
ts.append((z,i))
vivod(u'vibor_iwi1: z=%d jaz=%s\n'%(z,tpl[i].site().lang))
ts.sort()
ntpl=[]
n=0
for z,i in ts:
ntpl.append(tpl[i])
vivod(u'vibor_iwi2: z=%d jaz=%s\n'%(z,tpl[i].site().lang))
n+=1
if n>=nmax:
vivod(u'vibor_iwi: otbr!\n')
break
return ntpl
def rabota(slov,nscom,nssvoj,zapriz,tbl_pl,plprov,fproviz):
si0=plprov.site()
pl=wikipedia.Page(si0, plprov.title())
vivod(u'%s\n'%pl.title())
try:
plprov=get_tbl(tbl_pl,plprov)
ot = plprov.get()
except wikipedia.NoPage:
b = u'- %s\n' % plprov.title()
vivod(b)
return
except wikipedia.IsRedirectPage:
b = u'+ %s\n-> %s\n' % (plprov.title(), plprov.getRedirectTarget())
vivod(b)
return
if len(plprov.interwiki())<1:
vivod(u'-iwi\n')
return
try:
ot = pl.get()
except wikipedia.NoPage:
b = u'-2 %s\n' % pl.title()
vivod(b)
return
except wikipedia.IsRedirectPage:
b = u'+2 %s\n-> %s\n' % (pl.title(),pl.getRedirectTarget())
vivod(b)
return
otch.write(u'+ %s\n' % pl.title())
otch.flush()
otch2.write(u'%s\n-=======\n%s\n========\n' % (pl.title(),ot))
otch2.flush()
oiwi = pl.interwiki()
niwi = {}
for pl2 in oiwi:
if pl.site()!=pl2.site():
niwi[pl2.site()] = pl2
if len(niwi)<1:
vivod(u'-iwi2\n')
return
plkat = pl.categories()
nt = wikipedia.removeLanguageLinks(ot)
# nt = wikipedia.removeCategoryLinks(nt,pl.site())
nt = nt.replace(u'\r',u'')
while u' \n' in nt:
nt = nt.replace(u' \n',u'\n')
if len(nt)<1 or nt[len(nt)-1]!=u'\n':
nt+=u'\n'
fzam=0
jaz0=pl.site().lang
ntbk=ubrkoment(nt)
shapam=pl.site().family.disambig(jaz0)
fapa=0
for shapa in shapam:
if (iskats(ot,0,u'{{'+shapa+'}}')!=-1 or
iskats(ot,0,u'{{'+shapa[0].upper()+shapa[1:]+'}}')!=-1):
fapa=1
break
if fapa:
b = u'apa %s\n' % pl.title()
vivod(b)
return
res0=image_podp(pl.site(),ntbk)
vivod(u'len(res0)=%d\n'%len(res0))
kati=[]
for pka in plkat:
kati.append(Kateg(jaz0,pka.title(),1))
nomio=[(pl.title(),jaz0)]
shcomi=[]
eksli=[]
ssilki=[]
shcomi=isk_shabl(jaz0,ntbk,(u'Commons',u'Commonscat'))
for sh in shcomi:
korr_com_shabl(sh)
if not fproviz:
# eksli=isk_eksl(jaz0,ntbk)
ssilki=isk_ssilk(pl.site(),ntbk)
reso=[]
shcomo=[]
ekslo=[]
ssilko=[]
tablo=[]
kato=[]
itbot = imagetransfer1.ImageTransferBot(
targetSite = wikipedia.getSite(),otch=otch,opis=otch2,
otchnc=otchnc,nscom=nscom,otchkc=otchkc,otchup=otchup)
itbot.tbl_pl=tbl_pl
# for j in range(len(res0)):
# res0[j].tip=10
# iz=res0[j].iz
# b2=res0[j].podp
# b2=b2.replace(u'\n',u' ')
# b = u' b0 %s %s\n' % (iz,b2)
# vivod(b)
# itbot.zagruz_info(res0,1)
# if fproviz:
# fnajd=0
# itbot.prov_nekatco(res0)
# for j in range(len(res0)):
# if res0[j].nekatco:
# fnajd=1
# if not fnajd:
# return
niwisp=[]
for pli in niwi.itervalues():
niwisp.append(pli)
for pli in vibor_iwi(niwisp):
# if pli.site().lang!=u'en' and pli.site().lang!=u'hr':
# continue
try:
# get1(pli)
pli=get_tbl(tbl_pl,pli)
ti = pli.get()
except wikipedia.NoPage:
b = u' - %s:%s\n' % (pli.site().lang,pli.title())
vivod(b)
continue
except wikipedia.IsRedirectPage:
b = u' + %s:%s\n -> %s\n' % (pli.site().lang,pli.title(),pli.getRedirectTarget())
vivod(b)
pli = wikipedia.Page(pli.site(),pli.getRedirectTarget())
try:
# get1(pli)
pli=get_tbl(tbl_pl,pli)
ti = pli.get()
except wikipedia.NoPage:
b = u' - %s:%s\n' % (pli.site().lang,pli.title())
vivod(b)
continue
except wikipedia.IsRedirectPage:
b = u' + %s:%s\n -> %s\n' % (pli.site().lang,pli.title(),pli.getRedirectTarget())
vivod(b)
continue
if pli.isCategory():
b = u' +kat %s:%s\n' % (pli.site().lang,pli.title())
vivod(b)
continue
if pli.isImage():
b = u' +bil %s:%s\n' % (pli.site().lang,pli.title())
vivod(b)
continue
b = u' + %s:%s\n' % (pli.site().lang,pli.title())
vivod(b)
ti=ubrkoment(ti)
ti = ti.replace(u'\r',u'')
while u' ' in ti:
ti = ti.replace(u' ',u' ')
while u' \n' in ti:
ti = ti.replace(u' \n',u'\n')
(p0,p1)=isk_zagolov(ti,pli.site().lang,t_zag_vian)
p2=isk_kon_zagolov(ti,p0,p1)
plikat = pli.categories()
for pka in plikat:
kato.append(Kateg(pli.site().lang,pka.title(),1))
nomio.append( (pli.title(),pli.site().lang) )
shcoma=isk_shabl(pli.site().lang,ti,(u'Commons',u'Commonscat'))
for sh in shcoma:
korr_com_shabl(sh)
shcomo.append(sh)
if not fproviz:
if len(res0)==0:
res1=image_podp(pli.site(),ti)
for iz in res1:
reso.append(iz)
# eksla=isk_eksl(pli.site().lang,ti)
# for eksl in eksla:
# ekslo.append(eksl)
ssilka=isk_ssilk_smt(pli.site(),ti[p1:p2])
for ssilk in ssilka:
ssilko.append(ssilk)
# tabla=isk_tabl(pli.site().lang,ti)
# for tabl in tabla:
# tablo.append(tabl)
nnov_shcom=0
nnov_eksl=0
nnov_ssilk=0
nnov_tabl=0
nnov_kat=0
ntdi=u''
nnovdi_eksl=0
sico=wikipedia.getSite(code = u'commons',fam = u'commons')
nt_shcom=u''
shcom_prov_uzxe_povt(shcomi,shcomo)
if len(shcomo)>0:
nkpshpl=[]
for sh in shcomo+shcomi:
if sh.neisp:
continue
sh.pl=wikipedia.Page(sico,sh.arg[0])
nkpshpl.append(sh.pl)
sozd_buf_tbl(tbl_pl,nkpshpl,otch)
for sh in shcomo+shcomi:
if sh.neisp:
continue
try:
sh.pl=get_tbl(tbl_pl,sh.pl)
sh.pl.get()
b = u' + shcom:%s\n' % sh.pl.title()
vivod(b)
sh.arg[0]=sh.pl.title()
except wikipedia.NoPage:
b = u' - shcom:%s\n' % sh.pl.title()
vivod(b)
sh.neisp=1
except wikipedia.IsRedirectPage:
b = u' + shcom:%s\n -> %s\n' % (sh.pl.title(),
sh.pl.getRedirectTarget())
vivod(b)
sh.arg[0]=sh.pl.getRedirectTarget()
shcom_prov_uzxe_povt(shcomi,shcomo)
for j in range(len(shcomo)):
if shcomo[j].neisp:
continue
nt_shcom+= u'%s\n'%shcomo[j].tekstn(1)
nnov_shcom+=1
fzam=1
if nnov_shcom:
nt_shcom+= u'<br clear=all>\n'
nt_tabl=u''
for j in range(len(tablo)):
ta=tablo[j]
tat=ta.tekst().replace(u'\n',u' ')
vivod(u' tabl: %s\n'%tat)
if prov_tekst_uzxe(nt,ta.tekst()):
vivod(u' <-uzxe tabl\n')
continue
fpovt=0
for k in range(j):
if not ta.sravn(tablo[k]):
fpovt=1
break
if fpovt:
vivod(u' <-povt tabl\n')
continue
tap=perevod_ob(slov,ta.jaz,jaz0,ta.tekst()).t
nt_tabl+= u'<!-- %s -->\n'%tap
nnov_tabl+=1
fzam=1
for j in range(len(ekslo)):
eksl=ekslo[j]
ekslo[j].perpodp=perevod_ob(slov,eksl.jaz,jaz0,eksl.podp)
ekslo[j].perpodp2=Perev_podp(u'')
nt_eksl=u''
ntdi_eksl=u''
for j in range(len(ekslo)):
ekslo[j].tip=0
eksl=ekslo[j]
perpodp=eksl.perpodp
njazisp=eksl.njazisp
# vivod(u' %s\n'%eksl.tekst())
vivod(u' %s %d %s\n'%(eksl.jaz,eksl.njazisp,eksl.adr))
fuzxe=0
for eksl0 in eksli:
if not eksl.sravn(eksl0):
fuzxe=1
break
if fuzxe:
ekslo[j].tip=80
vivod(u' <-uzxe eksl\n')
continue
if zapriz.has_key(eksl.adr):
ekslo[j].tip=80
vivod(u' <-zapr\n')
continue
fpovt=0
for k in range(len(ekslo)):
if (k!=j) and not eksl.sravn(ekslo[k]):
if k<j:
fpovt=1
break
perpodp1=ekslo[k].perpodp
perpodp=objed_podp(perpodp,perpodp1)
njazisp+=ekslo[k].njazisp
if fpovt:
ekslo[j].tip=80
vivod(u' <-povt eksl\n')
continue
ekslo[j].perpodp2=perpodp
ekslo[j].njazisp2=njazisp
for j in range(len(ekslo)):
ekslo[j].perpodp=ekslo[j].perpodp2
ekslo[j].njazisp=ekslo[j].njazisp2
if ekslo[j].tip<80:
ekslo[j].prioritet=ekslo[j].njazisp
ekslo.sort(key=Eksl.vz_prioritet,reverse=True)
maksn_eksl=4
for j in range(len(ekslo)):
if ekslo[j].tip>=80:
continue
if ekslo[j].perpodp.t!=u'':
ekslo[j].perpodp.t=u'<!-- '+ekslo[j].perpodp.t+u' -->'
nt1= u'{{el}} %s\n'%ekslo[j].pertekst()
if j+len(eksli)<maksn_eksl:
nt_eksl+=nt1
nnov_eksl+=1
fzam=1
else:
# ntdi_eksl+=nt1
# nnovdi_eksl+=1
pass
for j in range(len(ssilko)):
ss=ssilko[j]
pe=perevod.perevod_iwi(slov,ss.jaz,jaz0,ss.ss)
vivod(u' ss %s:%s -> %s\n'%(ss.jaz,ss.ss,pe))
ssilko[j].ss=pe
nt_ssilk=u''
for j in range(len(ssilko)):
ss=ssilko[j]
if ss.ss==u'':
continue
vivod(u' %s\n'%ss.ss)
fuzxe=0
for ss0 in ssilki:
if iskats(ss.ss,0,u'[['+ss0.ss+u']]')!=-1:
fuzxe=1
break
if fuzxe:
vivod(u' <-uzxe ssilk\n')
continue
fpovt=0
for k in range(j):
if not ss.sravn(ssilko[k]):
fpovt=1
break
if fpovt:
vivod(u' <-povt ssilk\n')
continue
nt_ssilk+= u'* %s\n'%ss.ss
nnov_ssilk+=1
fzam=1
for j in range(len(reso)):
reso[j].perpodp2=Perev_podp(u'')
if reso[j].tip>=80:
continue
iz=reso[j].iz
if ((len(iz)>=6 and iz[0:2].isdigit() and iz[2:5]==u'px-') or
(len(iz)>=7 and iz[0:3].isdigit() and iz[3:6]==u'px-')):
reso[j].tip=90
vivod(u' - px- %s\n'%iz)
continue
ik=iskato(iz,len(iz)-1,u'.')
if ik==-1:
ik=len(iz)
izi=iz[:ik]
nalf=0
ncif=0
for c in izi:
if c.isdigit():
ncif+=1
# if c.isalpha():
else:
nalf+=1
if ncif>=4 and ncif>=nalf-2:
reso[j].tip=80
vivod(u' - cif %s\n'%iz)
continue
podp=reso[j].podp
jaz=reso[j].jaz
podp=podp.replace(u'\n',u' ')
while u' ' in podp:
podp = podp.replace(u' ',u' ')
if podp==u' ':
podp=u''
perpodp=perevod_ob(slov,jaz,jaz0,podp)
reso[j].perpodp=perpodp
for j in range(len(reso)):
reso[j].tip2=reso[j].tip
if reso[j].tip>=80:
continue
iz=reso[j].iz
perpodp=reso[j].perpodp
jaz=reso[j].jaz
njazisp=reso[j].njazisp
b = u' b %s:%s %s\n' % (jaz,iz,podp)
vivod(b)
if zapriz.has_key(iz):
reso[j].tip2=80
vivod(u' <-zapr\n')
continue
fuzxe=0
for k in range(len(res0)):
if res0[k].tip<80 and res0[k].iz==iz:
fuzxe=1
break
if fuzxe:
reso[j].tip2=80
vivod(u' <-uzxe\n')
continue
fpovt=0
k=0
while k<len(reso):
if reso[k].tip<80 and (k!=j) and (reso[k].iz==iz):
if k<j:
fpovt=1
break
perpodp1=reso[k].perpodp
perpodp=objed_podp(perpodp,perpodp1)
njazisp+=reso[k].njazisp
if perpodp1.t!=u'':
b = u' %s\n' % (perpodp1.t)
vivod(b)
k+=1
if fpovt:
reso[j].tip2=80
vivod(u' <-povt\n')
continue
reso[j].perpodp2=perpodp
reso[j].njazisp2=njazisp
for j in range(len(reso)):
reso[j].tip=reso[j].tip2
reso[j].perpodp=reso[j].perpodp2
reso[j].njazisp=reso[j].njazisp2
itbot.iskat(reso)
itbot.zagruz_info(reso,1)
kprefi=pl.site().family.image_namespace(pl.site().lang)
for j in range(len(reso)):
reso[j].tip2=reso[j].tip
if reso[j].tip>=80:
continue
iz=reso[j].iz
if (
#reso[j].tip>=50 and
(iskat(iz,u'flag')!=-1 or iskat(iz,u'Flag')!=-1 or
iskat(iz,u'FLAG')!=-1 or
iskat(iz,u'bandiera')!=-1 or
iskat(iz,u'Bandiera')!=-1 or
iskat(iz,u'BANDIERA')!=-1)):
reso[j].tip2=80
vivod(u' - flag %s\n'%iz)
continue
perpodp=reso[j].perpodp
njazisp=reso[j].njazisp
fuzxe=0
for k in range(len(res0)):
if res0[k].tip<80 and (res0[k].iz==iz or res0[k].dl==reso[j].dl):
fuzxe=1
break
if fuzxe:
reso[j].tip2=80
vivod(u' <-uzxe2\n')
continue
fpovt=0
for k in range(len(reso)):
if (reso[k].tip<80 and (k!=j) and
(reso[k].iz==iz or reso[k].dl==reso[j].dl)):
if reso[k].tip<reso[j].tip or (reso[k].tip==reso[j].tip and k<j):
fpovt=1
break
if reso[k].iz==iz:
perpodp1=reso[k].perpodp
perpodp=objed_podp(perpodp,perpodp1)
njazisp+=reso[k].njazisp
if fpovt:
reso[j].tip2=80
vivod(u' <-povt2\n')
continue
reso[j].perpodp2=perpodp
reso[j].njazisp2=njazisp
for j in range(len(reso)):
reso[j].tip=reso[j].tip2
reso[j].perpodp=reso[j].perpodp2
reso[j].njazisp=reso[j].njazisp2
if reso[j].tip<80:
reso[j].prioritet=reso[j].njazisp*1000+1000-reso[j].tip
reso.sort(key=imagetransfer1.Izobr.vz_prioritet,reverse=True)
for j in range(len(reso)):
vivod(u' [[%s:%s]] dl=%d nj=%d tip=%d\n'%
(kprefi,reso[j].iz,reso[j].dl,reso[j].njazisp,reso[j].tip))
maksn_iz=4
jn=0
for j in range(len(reso)):
if reso[j].tip>=80:
continue
if reso[j].tip>=50 and jn+len(res0)>=maksn_iz:
reso[j].tip=90
vivod(u'-otbr %s\n'%reso[j].iz)
jn+=1
flkatcom=0
for j in range(len(reso)):
if reso[j].tip==50:
flkatcom=1
itbot.fkatcom_ns=0
if flkatcom or fproviz:
katcomsp=[]
katprob=[]
for (n,jaz) in nomio:
katprob.append( (n,jaz) )
prov_dobav_katcom(nscom,zapriz,katcomsp,n,jaz)
if len(katcomsp)<1:
shcom_v_katcom(nscom,zapriz,katcomsp,shcomi+shcomo)
if len(katcomsp)<1:
for ka in kati+kato:
katprob.append( (ka.n,ka.jaz) )
prov_dobav_katcom(nscom,zapriz,katcomsp,ka.n,ka.jaz)
if len(katcomsp)<1:
vivod(u'- katcom\n')
for (n1,jaz) in katprob:
n2=n1.replace(u'(',u' ').replace(u')',u' ')
for n in n2.split(u' '):
if len(n)>=4:
prov_dobav_katcom(nscom,zapriz,katcomsp,n,jaz)
if len(katcomsp)<1:
vivod(u'- katcom2\n')
itbot.fkatcom_ns=1
for (n,jaz) in katprob:
t=u'Category:'+n
t1=u'[['+t+u']]\n'
if (not t1 in katcomsp) and (not zapriz.has_key(n)):
katcomsp.append(t1)
katcom=u''
for t in katcomsp:
katcom+=t
itbot.katcom=katcom
vivod(u'========katcom=\n%s==========\n' % katcom)
if itbot.fkatcom_ns:
for j in range(len(reso)):
if reso[j].tip==50:
reso[j].tip=60
if not fproviz:
itbot.kopir_izobr(reso)
nnov_iz=0
nnovdi_iz=0
jn=0
nt_iz=u''
ntdi_iz=u''
nt_izga=u'<gallery>\n'
ntdi_izga=u'<gallery>\n'
for j in range(len(reso)):
if reso[j].tip>=80:
continue
t=u''
if reso[j].perpodp.t!=u'':
t=u'|<!-- %s -->' % reso[j].perpodp.t
nt1= u'[[%s:%s|thumb|left|180px%s]]\n'%(kprefi,reso[j].iz,t)
nt2=(u'%s:%s|<!-- thumb|left|180px|%s -->\n'%
(kprefi,reso[j].iz,reso[j].perpodp.t))
if jn+len(res0)<maksn_iz:
nt_iz+=nt1
nt_izga+=nt2
nnov_iz+=1
fzam=1
else:
ntdi_iz+=nt1
ntdi_izga+=nt2
nnovdi_iz+=1
jn+=1
nt_izga+=u'</gallery>\n'
ntdi_izga+=u'</gallery>\n'
if nnov_iz>2:
nt_iz=nt_izga
if nnovdi_iz>2:
ntdi_iz=ntdi_izga
if nnov_iz:
nt_iz+= u'<br clear=all>\n'
nt+=nt_tabl
nt+=nt_iz
if len(nt_ssilk)>5:
(pzssilk0,pzssilk1)=isk_zagolov(nt,jaz0,t_zag_vian)
if pzssilk0==pzssilk1:
nt+=u'\n== '+gl_zagolov(jaz0,t_zag_vian)+u' ==\n'
pzssilk1=len(nt)
nt=nt[:pzssilk1]+nt_ssilk+nt[pzssilk1:]
nt+=nt_shcom
if len(nt_eksl)>5:
(pzeksl0,pzeksl1)=isk_zagolov(nt,jaz0,t_zag_eksl)
if pzeksl0==pzeksl1:
nt+=u'\n== '+gl_zagolov(jaz0,t_zag_eksl)+u' ==\n'
pzeksl1=len(nt)
nt=nt[:pzeksl1]+nt_eksl+nt[pzeksl1:]
if nnovdi_iz>0:
nt_shalka=u'{{%s 2}}'%wikipedia.translate(si0,te_al_bildoj)
if iskat(nt,nt_shalka)==-1:
nt+=nt_shalka+u'\n'
ntdi+=u'\n== %s ==\n'%wikipedia.translate(si0,te_al_bildoj)+ntdi_iz
fzam=1
if nnovdi_eksl>0:
nt_shalel=u'{{%s 2}}'%wikipedia.translate(si0,te_al_ekslj)
if iskat(nt,nt_shalel)==-1:
nt+=nt_shalel+u'\n'
ntdi+=u'\n== %s ==\n'%wikipedia.translate(si0,te_al_ekslj)+ntdi_eksl
fzam=1
nt_kat_sp=[]
glb=0
if len(kati)==0:
nt_kat_sp=perev_kateg(slov,nssvoj,pl.site(),kato,1,otch)
nnov_kat=len(nt_kat_sp)
if nnov_kat>0:
fzam=1
nt_kat=kateg_v_tekst(nt_kat_sp)
nt+=nt_kat
if fproviz:
for j in range(len(res0)):
# if res0[j].tip>=80 or res0[j].nekatco:
if res0[j].nekatco:
tc=res0[j].plcot+u'\n\n'+katcom
otch2.write(u'%s\n+=======\n%s\n====================================\n' % (res0[j].plco.title(),tc))
otch2.flush()
res0[j].plco.put(tc,comment = u'', minorEdit = False)
vivod(u'++ kat -> %s\n' % res0[j].plco.title())
if not fnajd:
return
if (not fproviz) and fzam:
if nt[len(nt)-1:]==u'\n':
nt=nt[:len(nt)-1]
# nt_stkat=wikipedia.replaceCategoryLinks(u'', ocat, pl.site())
# nt+=wikipedia.replaceLanguageLinks(nt_stkat, niwi)
nt+=u'\n'+wikipedia.replaceLanguageLinks(u'', niwi)
otch2.write(u'%s\n+=======\n%s\n====================================\n' % (pl.title(),nt))
otch2.flush()
t = u"AAld:"
if nnov_iz>0:
t+=u' + %d %s'%(nnov_iz,wikipedia.translate(si0,te_bildoj))
if nnov_shcom>0:
t+=u' + %d %s'%(nnov_shcom,wikipedia.translate(si0,te_komunejoj))
if nnov_eksl>0:
t+=u' + %d %s'%(nnov_eksl,wikipedia.translate(si0,te_ekslj))
if nnov_ssilk>0:
t+=u' + %d %s'%(nnov_ssilk,wikipedia.translate(si0,te_ligoj))
if nnov_tabl>0:
t+=u' + %d %s'%(nnov_tabl,wikipedia.translate(si0,te_tabeloj))
if nnov_kat>0:
t+=u' + %d %s'%(nnov_kat,wikipedia.translate(si0,te_kategorioj))
if nnovdi_iz>0 or nnovdi_eksl>0:
t+=u' + (%s)'%wikipedia.translate(si0,te_vi_di)
vivod(t+u'\n')
try:
put_prov(pl,nt,t,False)
except wikipedia.EditConflict:
vivod('!!!EditConflict!!!\n')
except wikipedia.LockedPage:
vivod('!!!LockedPage!!!\n')
if (not fproviz) and (nnovdi_iz>0 or nnovdi_eksl>0):
# diprefi=pl.site().family.namespace(pl.site().lang, 1, '_default')
# pldi=wikipedia.Page(pl.site(),diprefi+u':'+pl.title())
pldi=wikipedia.Page(pl.site(),pl.title()+u'/'+
wikipedia.translate(si0,te_aldon))
try:
tdi = pldi.get()
vivod(u'+ %s\n' % pldi.title())
except wikipedia.NoPage:
tdi=u''
vivod(u'- %s\n' % pldi.title())
except wikipedia.IsRedirectPage:
tdi=u'--> [['+pldi.getRedirectTarget()+u']]\n----\n'
vivod(u'+ %s\n-> %s\n' % (pldi.title(),pldi.getRedirectTarget()))
ntdi+=u'\n[[%s]]\n'%wikipedia.translate(si0,te_aldon_kat)
tdi+=u'\n'+ntdi
otch2.write(u'%s\n+=======\n%s\n====================================\n' % (pldi.title(),ntdi))
otch2.flush()
# t = u"AAld (%s):"%wikipedia.translate(si0,te_diskuto)
t = u"AAld (%s):"%(wikipedia.translate(si0,te_aldon).lower())
if nnovdi_iz>0:
t+=u' + %d %s'%(nnovdi_iz,wikipedia.translate(si0,te_bildoj))
if nnovdi_eksl>0:
t+=u' + %d %s'%(nnovdi_eksl,wikipedia.translate(si0,te_ekslj))
vivod(t+u'\n')
try:
pldi.put(tdi, comment = t, minorEdit = False)
except wikipedia.EditConflict:
vivod('!!!EditConflict!!!\n')
except wikipedia.LockedPage:
vivod('!!!LockedPage!!!\n')
return
def zapis_fimen(fimen,nomerf,pl):
fimen.write(u'%d %s\n' % (nomerf,pl.title()))
fimen.flush()
def provstop(fstop):
return os.access(fstop,os.F_OK)
def ivsenmain():
fstop='ivsenstop.txt'
try:
os.unlink(fstop)
except:
pass
if provstop(fstop):
wikipedia.output(u'ivsenstop0!\n')
return
n=u''
skon=u''
nomerf=-1
f=codecs.open(filename+'.txt','rb',encoding='utf-8')
frl=f.readlines()
mysite = wikipedia.getSite()
slov={}
nscom={}
nssvoj={}
zapriz={}
zaprobr={}
fproviz=0
i=0
j=0
fl1=1
for s in frl:
# wikipedia.output(u'%d\n' % ord(s[0]))
if ord(s[0]) == 65279:
s=s[1:]
# wikipedia.output(u'%d %d %s %s\n' % (i,j,n,s))
s=s.replace(u'\r',u'')
if s[len(s)-1]==u'\n':
s=s[:len(s)-1]
if fl1:
if s==u'':
fproviz=1
else:
slovdop={}
perevod.zagruzslov(slov,slovdop,nssvoj,u'',mysite.lang,s)
if slovdop.has_key(u'c'):
nscom=slovdop[u'c']
if slovdop.has_key(u'z'):
zapriz=slovdop[u'z']
if slovdop.has_key(u'zo'):
zaprobr=slovdop[u'zo']
fl1=0
else:
if len(s)>=1:
skon=s
frl=[]
f.close()
nomerf=0
i=0
while (i<len(skon)) and provcifr(skon[i]):
nomerf=nomerf*10+ord(skon[i])-ord(u'0')
i+=1
i=iskat(skon,u' ')
if i!=-1:
nomsta=skon[i+1:]
else:
nomsta=u'!'
nomerf+=1
global otch,otch2,otchnc,otchkc,otchup
otch = codecs.open(u'%s_ot%06d.txt' % (filename,nomerf), 'w', 'utf-8')
otch2 = codecs.open(u'%s_op%06d.txt' % (filename,nomerf), 'w', 'utf-8')
otchnc = codecs.open(u'%s_nc%06d.txt' % (filename,nomerf), 'w', 'utf-8')
otchkc = codecs.open(u'%s_kc%06d.txt' % (filename,nomerf), 'w', 'utf-8')
otchup = codecs.open(u'%s_up%06d.txt' % (filename,nomerf), 'w', 'utf-8')
fimen=codecs.open(filename+'.txt','a',encoding='utf-8')
otch.write(u'fproviz=%d\n\n'%fproviz)
otch.flush()
paggen=pagegenerators.AllpagesPageGenerator(nomsta, 0)
flperv=1
nkp=[]
maxnkp=16
for pla in paggen:
if flperv==1:
vivod(u'-uzxe %s\n'%pla.title())
flperv=0
continue
if zaprobr.has_key(pla.title()):
vivod(u'-zaprobr %s\n'%pla.title())
continue
nkp.append(pla)
if len(nkp)<maxnkp:
continue
tbl={}
sozd_buf_tbl(tbl,nkp,otch)
nkpi=[]
tdliz=[]
for pl in nkp:
try:
pl=get_tbl(tbl,pl)
ot = pl.get()
pl_iz=image_podp(pl.site(),ubrkoment(ot))
vivod(u'%s len(pl_iz)=%d\n'%(pl.title(),len(pl_iz)))
for pli in vibor_iwi(pl.interwiki()):
nkpi.append(pli)
if len(pl_iz)==0:
tdliz.append(pli)
except wikipedia.NoPage:
pass
except wikipedia.IsRedirectPage:
pass
sozd_buf_tbl(tbl,nkpi,otch)
tiz=[]
for pl in tdliz:
try:
pl=get_tbl(tbl,pl)
ot = pl.get()
tiz+=image_podp(pl.site(),ubrkoment(ot))
except wikipedia.NoPage:
pass
except wikipedia.IsRedirectPage:
pass
nkpiz=[]
sico=wikipedia.getSite(code = u'commons',fam = u'commons')
msprefi=mysite.family.image_namespace(mysite.lang)
for izo in tiz:
ctit=u'Image:'+izo.iz
if nscom.has_key(ctit):
vivod(u'tc+ %s\n' % ctit)
else:
nkpiz.append(wikipedia.Page(sico,ctit))
sourceSite=wikipedia.getSite(izo.jaz,fam = mysite.family)
isprefi=sourceSite.family.image_namespace(izo.jaz)
nkpiz.append(wikipedia.Page(sourceSite,isprefi+u':'+izo.iz))
nkpiz.append(wikipedia.Page(mysite,msprefi+u':'+izo.iz))
sozd_buf_tbl(tbl,nkpiz,otch)
vivod(u'\n\n********************************\n\n\n')
for pl in nkp:
rabota(slov,nscom,nssvoj,zapriz,tbl,pl,fproviz)
otch.write(u'\n\n\n')
otch.flush()
zapis_fimen(fimen,nomerf,pl)
vivod(u'\n\n********************************\n\n\n')
nkp=[]
if provstop(fstop):
wikipedia.output(u'ivsenstop!\n')
return
filename = wikipedia.argHandler(sys.argv[1], 'cht_cat')
#import logger
#sys.stdout = logger.Logger(sys.stdout, filename = u'%s_lo%06d.txt' % (filename,nomerf))
#mysite = wikipedia.getSite()
try:
# int(u'ert')
ivsenmain()
except:
wikipedia.stopme()
raise
else:
wikipedia.stopme()
</nowiki>* imagetransfer1.py
# -*- coding: utf-8 -*-
__version__='$Id: imagetransfer.py,v 1.44 2005/08/14 19:47:52 wikipedian Exp $'
import re, sys, md5, urllib, codecs
import wikipedia, upload1, config, pagegenerators, family
from bib_tekst import *
from get_buf import *
copy_message = {
'en':u"This image was copied from %s. The original description was:\r\n\r\n%s",
'eo':u"La bildo estas kopiita de %s. La originala priskribo estas:\r\n\r\n%s",
'de':u"Dieses Bild wurde von %s kopiert. Die dortige Beschreibung lautete:\r\n\r\n%s",
'nl':u"Afbeelding gekopieerd vanaf %s. De beschrijving daar was:\r\n\r\n%s",
'pt':u"Esta imagem foi copiada de %s. A descrição original foi:\r\n\r\n%s",
'ru':u"Это изображение скопировано из %s. Его исходное описание:\r\n\r\n%s",
}
istf_message = {
'en':u"File history",
'eo':u"Historio de la dosiero",
'ru':u"История изменений файла",
}
#nowCommonsTemplate = {
# 'de': u'{{NowCommons|%s}}',
# 'fr': u'{{Désormais sur Commons|%s}}',
# 'en': u'{{NowCommons|Image:%s}}',
# 'nl': u'{{NuCommons|%s}}',
# 'pt': u'{{NowCommons|%s}}',
#}
nowCommonsTemplate = {
'ca': u'AraCommons',
'es': u'EnCommons',
'et': u'NüüdCommonsis',
'fr': u'Désormais sur Commons',
'nl': u'NuCommons',
'ru': u'Now Commons',
}
nowCommonsMessage = {
'de': u'Datei ist jetzt auf Wikimedia Commons verfügbar.',
'en': u'File is now available on Wikimedia Commons.',
'eo': u'La dosiero nun estas en Komunejo.',
'pt': u'Arquivo está agora na Wikimedia Commons.',
}
# Translations for license templates.
# Must only be given when they are in fact different.
licenseTemplates = {
('wikipedia:es', 'commons:commons'): {
u'DP': u'PD',
},
}
licenseTemplates_o = {
u'Bild-GFDL': u'GFDL',
u'Bild-GFDL-OpenGeoDB': u'GFDL-OpenGeoDB',
u'Bild-PD': u'PD',
u'Bild-PD-US': u'PD-USGov',
u'Domaine Public': u'PD',
u'Domaine public': u'PD',
u'DomainePublic': u'PD',
u'Domínio público': u'PD',
u'Eigenwerk': u'GFDL-self',
u'EigenWerk': u'GFDL-self',
u'Eigenwurk': u'GFDL-self',
u'EigenWurk': u'GFDL-self',
}
def ubrkoment (text):
# Ignore tekst within nowiki tags and HTML comments
nowikiOrHtmlCommentR = re.compile(r'<nowiki>.*?|<!--.*?-->', re.IGNORECASE | re.DOTALL)
match = nowikiOrHtmlCommentR.search(text)
while match:
text = text[:match.start()] + text[match.end():]
match = nowikiOrHtmlCommentR.search(text)
return text
def getVersionHistoryTable_ss(self):
"""
Returns the version history as a wiki table.
"""
uprefi=self.site().family.namespace(self.site().lang, 2, '_default')
result = '{| border="1"\n'
result += '! date/time || username || edit summary\n'
for time, username, summary in self.getVersionHistory():
result += '|----\n'
result += '| %s || [[:%s:%s:%s]] || %s\n' % (time, self.site().lang,uprefi,username, summary)
result += '|}\n'
return result
class Izobr:
def __init__(self, iz,jaz,podp):
iz = iz.replace(u'_', ' ')
iz = iz[0].upper()+iz[1:]
self.iz = iz
self.jaz = jaz
self.podp = podp
self.podp2 = u''
self.url=u''
self.dl=0 # dlina fajla
self.opis=u''
self.tip=70 # 10=uzxe v statje 20=uzxe v statje + oshibka chtenija
# 30=uzxe v commons 40=uzxe v nashem razdele
# 50=nado zagruz v commons 60=nado zagruz nash razdel
# 70=nado zagruz
# >=80 =nelzja ispolz (oshibka)
self.tip2=70
self.prioritet=0
self.njazisp=1
self.njazisp2=0
self.plco=None
self.plcot=u''
self.nekatco=0
def vz_prioritet(self):
return self.prioritet
def sravn(self,e1):
return self.iz!=e1.iz
# def tekst(self,pref):
# return u'%s %s'%(self.adr,self.podp)
#def stranica_url(pl):
def stranica_url(sourceSite,ti):
# sourceSite=pl.site()
# ti=pl.title()
# Spaces might occur, but internally they are represented by underscores.
# Change the name now, because otherwise we get the wrong MD5 hash.
encodedFilename = ti.replace(u' ', u'_').encode(sourceSite.encoding())
encodedFilename = urllib.quote(encodedFilename)
# TODO: This probably doesn't work on all wiki families
return 'http://%s/wiki/%s' % (sourceSite.hostname(),encodedFilename)
def url_soderzx(otch,url):
s=u''
try:
uo = wikipedia.MyURLopener()
file = uo.open(url)
info=file.info()
except IOError:
otch.write(u'url_soderzx: IOError\n')
otch.flush()
return u''
try:
s=file.read()
except IOError:
otch.write(u'url_soderzx: read IOError\n')
otch.flush()
return u''
otch.write(u'url_soderzx2 len(s)=%d\n'%len(s))
# if len(s)>=5:
# otch.write(u'url_soderzx2 s=%d %d %d %d %d\n'%(ord(s[0]),ord(s[1]),
# ord(s[2]),ord(s[3]),ord(s[4])))
otch.flush()
# if len(s)<10 or s.startswith('<html'):
# otch.write(u'url_soderzx3ne\n')
# otch.flush()
# return u''
return s
def izobr_url(sourceSite,im):
# Spaces might occur, but internally they are represented by underscores.
# Change the name now, because otherwise we get the wrong MD5 hash.
encodedFilename = im.replace(u' ', u'_').encode(sourceSite.encoding())
md5sum = md5.new(encodedFilename).hexdigest()
# if debug: print "MD5 hash is: %s" % md5sum
encodedFilename = urllib.quote(encodedFilename)
el = sourceSite.lang.encode(sourceSite.encoding())
el = urllib.quote(el)
# TODO: This probably doesn't work on all wiki families
# return 'http://%s/upload/%s/%s/%s' % (sourceSite.hostname(), md5sum[0], md5sum[:2], encodedFilename)
return ('http://upload.wikimedia.org/wikipedia/%s/%s/%s/%s' %
(el, md5sum[0], md5sum[:2], encodedFilename))
def url_dlina(otch,url):
dl=0
try:
uo = wikipedia.MyURLopener()
file = uo.open(url)
info=file.info()
except IOError:
return 0
dlt=info.getheader('Content-Length')
try:
dl=int(dlt)
except ValueError:
return 0
except TypeError:
return 0
otch.write(u'url_dlina1 dl=%s\n'%dl)
otch.flush()
try:
s=file.read(10)
except IOError:
return 0
otch.write(u'url_dlina2 len(s)=%d\n'%len(s))
if len(s)>=5:
otch.write(u'url_dlina2 s=%d %d %d %d %d\n'%(ord(s[0]),ord(s[1]),
ord(s[2]),ord(s[3]),ord(s[4])))
otch.flush()
if len(s)<10 or s.startswith('<html'):
otch.write(u'url_dlina3ne\n')
otch.flush()
return 0
return dl
class ImageTransferBot:
def __init__(self, targetSite, otch,opis,otchnc=None,
nscom=None,otchkc=None,otchup=None):
self.targetSite = targetSite
self.otch = otch
self.opis = opis
self.otchnc = otchnc
self.otchkc = otchkc
self.otchup = otchup
self.nscom = nscom
self.katcom = u''
self.fkatcom_ns = 0
self.sico=wikipedia.getSite(code = u'commons',fam = u'commons')
self.tbl_pl={}
def vivod(self,b):
wikipedia.output(b)
self.otch.write(b)
self.otch.flush()
def prov_nekatco1(self, izo):
sico=self.sico
izo.plco = wikipedia.Page(sico,u'Image:'+izo.iz)
izo.nekatco=0
try:
izo.plcot=izo.plco.get()
except wikipedia.NoPage:
self.vivod(u'c- %s\n' % izo.plco.title())
return
except wikipedia.IsRedirectPage:
izo.nekatco=1
izo.plcot=u'--> [['+izo.plco.getRedirectTarget()+u']]\n----\n'
self.vivod(u'c+ %s\n-> %s\n' % (izo.plco.title(),izo.plco.getRedirectTarget()))
self.vivod(u'<- nekat\n')
return
self.vivod(u'c+ %s\n' % izo.plco.title())
plkat = izo.plco.categories()
if len(plkat)<1:
izo.nekatco=1
self.vivod(u'<- nekat\n')
return
def podgot_istf(self,ishsi,s):
if len(s)<10:
self.vivod(u'!!!osh cht ist fajla!!!\n')
return u''
s=unicode(s,'utf-8','replace')
p=iskats(s,0,u'<h2 id=\"filehistory\">')
if p!=-1:
konz=u'</h2>'
p=iskats(s,p,konz)
if p!=-1:
p+=len(konz)
pk=iskats(s,p,u'<br /><ul><li>')
if pk==-1:
pk=iskats(s,p,u'<h2')
if pk!=-1:
s=s[p:pk]
else:
s=s[p:]
s=s.replace(u'\r',u'').replace(u'\n',u' ')
s=s.replace(u'<br>',u'\n\n')
s=s.replace(u'<br />',u'\n\n')
s=s.replace(u'<li>',u'\n* ')
fl=0
t=u'==== '+wikipedia.translate(self.targetSite,istf_message)+u' ====\n'
nachss=u'<a href=\"/wiki/'
i=0
while i<len(s):
c=s[i]
if s[i:].startswith(nachss):
p0=i+len(nachss)
p=iskats(s,p0,u'\"')
fl=1
if p!=-1:
ss=s[p0:p]
i=p
# ss=wikipedia.unescape(ss)
ss=ss.encode('utf-8','replace')
ss=urllib.unquote(ss)
ss=unicode(ss,'utf-8','replace')
t+=u' [[:%s:%s]] '%(ishsi.lang,ss)
# konss=u'</a>'
# p=iskats(s,i,konss)
# if p!=-1:
# i=p+len(konss)
# fl=0
elif c==u'<':
fl=1
elif c==u'>':
fl=0
elif fl==0:
t+=c
i+=1
t = t.replace(u'{',u'{ ')
t = t.replace(u'}',u' }')
while u' ' in t:
t = t.replace(u' ',u' ')
t = t.replace(u' \n',u'\n')
t = t.replace(u'\n ',u'\n')
return t
def iskat1(self, izo):
if izo.tip!=70:
return
im=izo.iz
targetSite=self.targetSite
sourceSite=wikipedia.getSite(izo.jaz,fam = targetSite.family)
isprefi=sourceSite.family.image_namespace(izo.jaz)
sourceImagePage=wikipedia.Page(sourceSite,isprefi+u':'+izo.iz)
# need to strip off "Afbeelding:", "Image:" etc.
# we only need the substring following the first colon
# im = sourceImagePage.title().split(":", 1)[1]
# im = im.replace(u'_', ' ')
if len(im)<1:
izo.tip=90
return
# Also, the first letter should be capitalized
# TODO: Don't capitalize on non-capitalizing wikis
im = im[0].upper()+im[1:]
sico=self.sico
b=u'\n0 %s\n' % sourceImagePage.title()
self.vivod(b)
tit=u'Image:'+im
if self.nscom and self.nscom.has_key(tit):
b=u'tc+ %s\n' % tit
self.vivod(b)
izo.tip=30
return
# urlp=izobr_url(targetSite,im)
# self.vivod(u'p %s\nurl=%s\n' % (im,urlp))
# dl=url_dlina(self.otch,urlp)
# self.vivod(u' Content-Length=%d\n'%dl)
# if dl>0:
# izo.dl=dl
# izo.tip=40
# self.vivod(u'dlu+ %s\n' % im)
# return
try:
# get1(sourceImagePage)
sourceImagePage=get_tbl(self.tbl_pl,sourceImagePage)
dcom = sourceImagePage.get()
except wikipedia.NoPage:
b=u'i- %s\n' % sourceImagePage.title()
self.vivod(b)
dcom=u''
except wikipedia.IsRedirectPage:
dcom=u''
for old, new in licenseTemplates_o.iteritems():
rnew = '{{%s}}' % new
rold = re.compile('{{%s}}' % old)
dcom = wikipedia.replaceExceptNowikiAndComments(dcom, rold, rnew)
old = old[0].lower()+old[1:]
rold = re.compile('{{%s}}' % old)
dcom = wikipedia.replaceExceptNowikiAndComments(dcom, rold, rnew)
if licenseTemplates.has_key((sourceSite.sitename(), 'commons:commons')):
for old, new in licenseTemplates[(sourceSite.sitename(), 'commons:commons')].iteritems():
rnew = '{{%s}}' % new
rold = re.compile('{{%s}}' % old)
dcom = wikipedia.replaceExceptNowikiAndComments(dcom, rold, rnew)
old = old[0].lower()+old[1:]
rold = re.compile('{{%s}}' % old)
dcom = wikipedia.replaceExceptNowikiAndComments(dcom, rold, rnew)
dcomnk=ubrkoment(dcom)
ncti=u'{{NowCommons|'
p=iskat(dcomnk,ncti)
if p==-1:
ncti=u'{{nowCommons|'
p=iskat(dcomnk,ncti)
if p==-1 and nowCommonsTemplate.has_key(sourceSite.lang):
t=nowCommonsTemplate[sourceSite.lang]
ncti=u'{{'+t+u'|'
p=iskat(dcomnk,ncti)
if p==-1:
ncti=u'{{'+perv_lower(t)+u'|'
p=iskat(dcomnk,ncti)
if p!=-1:
self.vivod(u'najden NowCommons\n')
p+=len(ncti)
p1=iskats(dcomnk,p,u'}}')
if p1!=-1:
p2=iskats(dcomnk[:p1],p,u'|')
if p2!=-1 and p2<p1:
p1=p2
imc=ubr_nk_prob(dcomnk[p:p1])
if imc.startswith(u':'):
imc=imc[1:]
# p=iskat(imc,u':')
# if p!=-1:
# imc=imc[p+1:]
if imc.startswith(u'Image:') or imc.startswith(u'image:'):
imc=imc[6:]
imc = ubr_nk_prob(imc.replace(u'_', u' '))
imc = imc[0].upper()+imc[1:]
tit=u'Image:'+imc
if self.nscom and self.nscom.has_key(tit):
b=u'tsc+ %s\n' % tit
self.vivod(b)
izo.iz=imc
izo.tip=30
return
plcoi = wikipedia.Page(sico,tit)
# get1(plcoi)
plcoi=get_tbl(self.tbl_pl,plcoi)
plcoie=plcoi.exists()
if plcoie:
b=u'sc+ %s\n' % tit
self.vivod(b)
izo.iz=imc
izo.tip=30
return
else:
b=u'sc- %s\n' % tit
self.vivod(b)
plco = wikipedia.Page(sico,u'Image:'+im)
# get1(plco)
plco=get_tbl(self.tbl_pl,plco)
plcoe=plco.exists()
if plcoe:
b=u'c+ %s\n' % plco.title()
self.vivod(b)
izo.tip=30
return
else:
b=u'c- %s\n' % plco.title()
self.vivod(b)
targetPr=self.targetSite.family.image_namespace(self.targetSite.language())
pl = wikipedia.Page(self.targetSite,wikipedia.UnicodeToAsciiHtml(targetPr+':'+im))
# get1(pl)
pl=get_tbl(self.tbl_pl,pl)
ple=pl.exists()
if ple:
b=u'u+ %s\n' % im
self.vivod(b)
izo.tip=40
return
# localize the text that should be printed on the image description page
dcom = wikipedia.translate(self.targetSite, copy_message) % (sourceSite, dcom)
sv_shabl=[u'GFDL',u'PD-',u'GPL',u'LGPL',u'Cc-by',u'Cc-sa',
u'CC-BY',u'CC-SA',u'CC|',
u'BSD}}',u'CeCILL}}',u'FWL}}',u'MPL}}',u'Self_GFDL',
u'Bild-PD-',u'Bild-GFDL',
# u'Bild-PD',u'PD',u'Domaine public',u'Domínio público',
# u'Public Domain',u'Public domain',
# u'Eigenwerk',u'Eigenwurk',u'EigenWerk',u'EigenWurk',
u'Bild-CC']
sv_tekst=[u'Lizenz: GFDL',
u'Lizenz: [[GFDL]]',
u'Lizenz: GNU FDL',
u'Lizenz: [[GNU FDL]]',
u'Lizenz: GNU-FDL',
u'Lizenz: [[GNU-FDL]]',
u'Lizenz: GPL',
u'Lizenz: [[GPL]]',
u'Lizenz: LGPL',
u'Lizenz: [[LGPL]]',
# u'Lizenz: PD',
# u'Lizenz: [[PD]]',
# u'Lizenz: Public Domain',u'Lizenz: public domain',
# u'Lizenz: [[Public Domain]]',
u'Lizenz: CC', u'Lizenz: Cc', u'Lizenz: cc',
u'Lizenz: [[CC]]',]
ne_shabl=[u'Copyvio',u'Fairuse',u'Fairusein',u'Logo',
u'CopyrightedFreeUse',
u'CopyrightedFreeUseProvidedThat',
u'Deletion request',
u'Screenshot',u'Noncommercial',u'PolandGov',
u'Nonderivative',u'NZCrownCopyright',
u'Redundant',u'No source',
u'Ordnance Survey Copyright',
u'CrownCopyright',
u'EU image',
u'Cc-nc',u'Cc-by-nc',u'Cc-nd',u'Cc-by-nd',
u'CC-NC',u'CC-BY-NC',u'CC-ND',u'CC-BY-ND',
]
sv_tekst2=[]
for z1 in sv_tekst:
sv_tekst2.append(z1)
sv_tekst2.append(z1.replace(u': ',u':'))
z2=z1.replace(u'Lizenz:',u'Lizenzstatus:')
sv_tekst2.append(z2)
sv_tekst2.append(z2.replace(u': ',u':'))
sv_shabl+=perv_lower_s(sv_shabl)
sv_tekst2+=perv_lower_s(sv_tekst2)
ne_shabl+=perv_lower_s(ne_shabl)
fsvob=0
for z1 in sv_shabl:
if iskat(dcomnk,u'{{'+z1)!=-1:
fsvob=1
for z1 in sv_tekst2:
if iskat(dcomnk,z1)!=-1:
fsvob=1
for z1 in ne_shabl:
if iskat(dcomnk,u'{{'+z1)!=-1:
fsvob=0
self.vivod(u'!!ne_shabl!!\n')
izo.tip=60
if fsvob:
izo.tip=50
self.vivod(u'++->c\n')
izo.opis=dcom
def zagruz_info1(self, izo, fco):
if izo.tip>=80:
return
im=izo.iz
targetSite=self.targetSite
sourceSite=wikipedia.getSite(izo.jaz,fam = targetSite.family)
if izo.dl<=0 and izo.tip!=30:
izo.url=izobr_url(sourceSite,im)
self.vivod(u'0 %s\nurl=%s\n' % (im,izo.url))
izo.dl=url_dlina(self.otch,izo.url)
self.vivod(u' Content-Length=%d\n'%izo.dl)
if izo.dl<=0 and fco:
self.vivod(u'iz c:\n')
izo.url=izobr_url(self.sico,im)
self.vivod(u'0 %s\nurl=%s\n' % (im,izo.url))
izo.dl=url_dlina(self.otch,izo.url)
self.vivod(u' Content-Length=%d\n'%izo.dl)
if izo.dl<=0:
izo.tip=90
izo.dl=0
def kopir_izobr1(self, izo):
if izo.tip!=50 and izo.tip!=60:
return
b=u'\n -> %s\n' % izo.iz
self.vivod(b)
sico=self.sico
targetSite = self.targetSite
sourceSite=wikipedia.getSite(izo.jaz,fam = targetSite.family)
isprefi=sourceSite.family.image_namespace(izo.jaz)
sourceImagePage=wikipedia.Page(sourceSite,isprefi+u':'+izo.iz)
# TODO: Only the page's version history is shown, but the file's
# version history would be more helpful
# izo.opis += '\n\n' + sourceImagePage.getVersionHistoryTable()
izo.opis += '\n\n' + getVersionHistoryTable_ss(sourceImagePage)
str_url=stranica_url(sourceSite,isprefi+u':'+izo.iz)
self.vivod(u'str_url=%s\n'%str_url)
istf=url_soderzx(self.otch,str_url)
istf=self.podgot_istf(sourceSite,istf)
izo.opis += u"\n" + istf
# add interwiki link
izo.opis += u"\n\n" + sourceImagePage.aslink(forceInterwiki = True)
t_ot_c=u''
if izo.tip==50:
targetSite = sico
if self.katcom!=u'':
izo.opis += '\n\n' + self.katcom
self.vivod(u'++->c\n')
self.otchnc.write(u'[[:%s:%s]]\n'%(sourceSite.lang,
sourceImagePage.title()))
self.otchnc.flush()
t=u''
if self.fkatcom_ns:
t=u'!!!!'
self.otchkc.write(u'Image:%s\n==========%s=\n%s===============\n'%
(izo.iz,t,self.katcom))
self.otchkc.flush()
t_ot_c=u'->c'
bot = upload1.UploadRobot(url = izo.url, description = izo.opis,
targetSite = targetSite, urlEncoding = sourceSite.encoding(),
otchup=self.otchup)
# try to upload
tosh=u''
(targetFilename,tosh) = bot.run()
if tosh!=u'':
self.vivod(u'!!!! osh -> povt %s %s\n'%(izo.iz,tosh))
(targetFilename,tosh) = bot.run()
if tosh!=u'':
self.vivod(u'!!!! osh !!!! %s %s\n'%(izo.iz,tosh))
izo.tip=90
else:
b=u'++ %s\n'% izo.iz
self.vivod(b)
b=(u'======zagr%s====\n%s\n========\n%s\n========================\n'%
(t_ot_c,izo.iz,izo.opis))
self.opis.write(b)
self.opis.flush()
return
def prov_nekatco(self,generator):
for izo in generator:
self.prov_nekatco1(izo)
def iskat(self,generator):
for izo in generator:
self.iskat1(izo)
def zagruz_info(self,generator,fco):
for izo in generator:
self.zagruz_info1(izo,fco)
def kopir_izobr(self,generator):
for izo in generator:
self.kopir_izobr1(izo)
def main():
pageTitle = []
page = None
gen = None
# fnscom = wikipedia.argHandler(sys.argv[1], 'cht_cat')
# filename = wikipedia.argHandler(sys.argv[2], 'cht_cat')
filename = wikipedia.argHandler(sys.argv[1], 'cht_cat')
mysite = wikipedia.getSite()
vihprefi=mysite.family.image_namespace(mysite.lang)
# gen = pagegenerators.TextfilePageGenerator(filename+'.txt')
f=codecs.open(filename+'.txt','rb',encoding='utf-8')
otch = codecs.open(u'%s_ot.txt' % filename, 'w', 'utf-8')
otch2 = codecs.open(u'%s_op.txt' % filename, 'w', 'utf-8')
otchnc = codecs.open(u'%s_nc.txt' % filename, 'w', 'utf-8')
otchkc = codecs.open(u'%s_kc.txt' % filename, 'w', 'utf-8')
otchup = codecs.open(u'%s_up.txt' % filename, 'w', 'utf-8')
otchiw = codecs.open(u'%s_iw.txt' % filename, 'w', 'utf-8')
katcom=u''
if len(sys.argv)>=3:
fnkc = wikipedia.argHandler(sys.argv[2], 'cht_cat')
fkc = codecs.open(fnkc, 'rb', 'utf-8')
katcom=fkc.read()
fkc.close()
nscom={}
# perevod.zagruz_ns(nscom,fnscom)
itbot = ImageTransferBot(
targetSite = mysite,otch=otch,opis=otch2,
otchnc=otchnc,nscom=nscom,otchkc=otchkc,otchup=otchup)
# reso = []
# for pl in gen:
# reso.append(Izobr(pl.title(),pl.site().lang,u''))
# for j in range(len(reso)):
# reso[j].tip=70
# for pl in gen:
# reso = []
# t=pl.title()
# p=iskat(t,u':')
# if p!=-1:
# t=t[p+1:]
# reso.append(Izobr(t,pl.site().lang,u''))
for s in f.readlines():
if ord(s[0]) == 65279:
s=s[1:]
itbot.katcom=katcom
im=s
p=iskat(s,u']]')
if p!=-1:
im=s[:p]
kc=ubr_nk_prob(s[p+2:])
if kc!=u'':
itbot.katcom=kc
if im.startswith(u'[['):
im=im[2:]
imt=im.split(u':',2)
if len(imt)<3:
continue
imt[2]=perv_upper(ubr_nk_prob(imt[2]))
reso=[Izobr(imt[2],imt[0],u'')]
reso[0].tip=70
sourceSite=wikipedia.getSite(imt[0],fam = mysite.family)
isprefi=sourceSite.family.image_namespace(imt[0])
otch.write(u'\n\n%s:%s:%s\n==katcom==\n%s\n==========\n'%
(imt[0],isprefi,imt[2],itbot.katcom))
otch.flush()
itbot.iskat(reso)
itbot.zagruz_info(reso,1)
if itbot.katcom==u'':
if reso[0].tip==50:
reso[0].tip=60
itbot.kopir_izobr(reso)
if reso[0].tip<80:
otchiw.write(u'%s:%s:%s\n%s:%s:%s\n\n'%
(mysite.lang,vihprefi,reso[0].iz,imt[0],isprefi,imt[2]))
otchiw.flush()
else:
otch.write(u'!!!- %s:%s:%s\n'%
(imt[0],isprefi,imt[2]))
otch.flush()
if __name__ == "__main__":
try:
main()
finally:
wikipedia.stopme()
</nowiki>* perevod.py
# -*- coding: UTF-8 -*-
__version__='$Id:'
import wikipedia
import re, sys
import codecs
from bib_tekst import *
def vivod(b):
# wikipedia.output(b)
# otch.write(b)
# otch.flush()
pass
class Sl_sta:
def __init__(self,vihs):
self.iwi = []
self.tp = [vihs]
self.riwi=0
def dobavslov2(slov,vhjaz,vihjaz,vhs,vihs,fiwi,fltolk):
if vhs==u'Circumscribed' or vhs==u'circumscribed':
a=-1
b=-1
if slov.has_key((vhjaz,vhs)):
a=len(slov[(vhjaz,vhs)].tp)
b=len(slov[(vhjaz,vhs)].iwi)
wikipedia.output(u'2 %s %s %d %d %d %d'%(vhs,vihs,fiwi,fltolk,a,b))
if slov.has_key((vhjaz,vhs)):
if fltolk==2:
return
if fltolk==1:
slov[(vhjaz,vhs)].tp=[vihs]
elif not vihs in slov[(vhjaz,vhs)].tp:
slov[(vhjaz,vhs)].tp.append(vihs)
if fiwi:
if not vihs in slov[(vhjaz,vhs)].iwi:
slov[(vhjaz,vhs)].iwi.append(vihs)
else:
slov[(vhjaz,vhs)]=Sl_sta(vihs)
if fiwi:
slov[(vhjaz,vhs)].iwi.append(vihs)
if fiwi==2:
slov[(vhjaz,vhs)].riwi=1
# i=len(slov[(vhjaz,vhs)].tp)-1
# otch.write(u'%d %s:%s:%s\n'%(i,vhjaz,vhs,vihs))
# otch.flush()
def eo_pluralo(t):
(p,n)=iskats_mn(t,0,[u' de ',u' en ',u' el ',u' al ',u' je ',u' por '])
if p==-1:
p=len(t)
p=t[:p].replace(u'a ',u'aj ')+'j'+t[p:]
# wikipedia.output(p)
return p
en_soglasn=[u'b',u'c',u'd',u'f',u'g',u'h',u'j',u'k',u'l',u'm',u'n',u'p',u'q',u'r',u's',u't',u'v',u'w',u'x',u'z']
def fleksii_en_eo_glag(slov,vhs,osneo,fltolk):
vhok=vhs[len(vhs)-1]
if vhok==u'e':
vhs1=vhs[:len(vhs)-1]+u'es'
vihs1=osneo+u'as'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs[:len(vhs)-1]+u'ed'
vihs1=osneo+u'is'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs[:len(vhs)-1]+u'ing'
vihs1=osneo+u'anta'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
elif vhok==u'y':
vhs1=vhs[:len(vhs)-1]+u'ies'
vihs1=osneo+u'as'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs+u'es'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs+u's'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs[:len(vhs)-1]+u'ied'
vihs1=osneo+u'is'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs+u'ed'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs[:len(vhs)-1]+u'ing'
vihs1=osneo+u'anta'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs+u'ing'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
else:
vhs1=vhs+u's'
vihs1=osneo+u'as'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs+u'ed'
vihs1=osneo+u'is'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
if vhok in en_soglasn:
vhs1=vhs+vhok+u'ed'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs+u'ing'
vihs1=osneo+u'anta'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
if vhok in en_soglasn:
vhs1=vhs+vhok+u'ing'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
def fleksii_en_eo(slov,vhs,vihs,fltolk):
# if (u' ' in vhs) or (u' ' in vihs):
# return
if len(vhs)<3 or len(vihs)<3:
return
tipeo=vihs[len(vihs)-1]
osneo=vihs[:len(vihs)-1]
vhok=vhs[len(vhs)-1]
if tipeo==u'o':
vhs1=vhs+u'\'s'
# vihs1=u'de '+vihs
vihs1=osneo+u'a'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vihs1=eo_pluralo(vihs)
if vhok==u'y':
vhs1=vhs[:len(vhs)-1]+u'ies'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs+u's'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs+u'es'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
osneo1=osneo
if osneo1[len(osneo1)-2:]==u'il':
osneo1=osneo1[:len(osneo1)-2]
if (not u' ' in vhs) and (not u' ' in vihs):
fleksii_en_eo_glag(slov,vhs,osneo1,fltolk)
elif tipeo==u'i':
fleksii_en_eo_glag(slov,vhs,osneo,fltolk)
elif tipeo==u'a':
vhs1=vhs+u'ly'
vihs1=osneo+u'e'
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs+u'er'
vihs1=u'pli '+vihs.lower()
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
vhs1=vhs+u'est'
vihs1=u'plej '+vihs.lower()
dobavslov2(slov,u'en',u'eo',vhs1,vihs1,0,fltolk)
def dobavslov(slov,vhjaz,vihjaz,vhs,vihs,fiwi,fltolk):
vhs=ubr_nk_prob(vhs)
if vhs==u'':
return
if vhs==u'Circumscribed' or vhs==u'circumscribed':
wikipedia.output(u'%s %s %d %d'%(vhs,vihs,fiwi,fltolk))
vihs=ubr_nk_prob(vihs)
if vihs==u'':
return
vhs=vhs[0].upper()+vhs[1:]
vihs=vihs[0].upper()+vihs[1:]
dobavslov2(slov,vhjaz,vihjaz,vhs,vihs,fiwi,fltolk)
if vhjaz==u'en' and vihjaz==u'eo':
fltolk1=fltolk
if fltolk1==1:
fltolk1=0
fleksii_en_eo(slov,vhs,vihs,fltolk1)
def zagruzslov_revo(slov,vhjaz,vihjaz,fslov,ftdob):
f=codecs.open(fslov,'rb',encoding='utf-8')
for s in f.readlines():
# wikipedia.output(u'%d\n' % ord(s[0]))
if ord(s[0]) == 65279:
s=s[1:]
# wikipedia.output(u'%d %d %s %s\n' % (i,j,n,s))
s=s.replace(u'\r',u'')
if s[len(s)-1]==u'\n':
s=s[:len(s)-1]
fltolk=0
if ftdob:
fltolk=2
elif len(s)>=1 and s[0]==u'!':
fltolk=1
s=s[1:]
i=iskat (s,u':')
if i!=-1:
vhs=s[:i]
vihst=s[i+1:].replace(u';',u',').split(u',')
if (u',' in vhs) or (u';' in vhs):
j=0
sk=0
vhst=[]
while j<len(vhs):
if vhs[j]==u'(' or vhs[j]==u'[':
sk+=1
elif vhs[j]==u')' or vhs[j]==u']':
sk-=1
if sk==0 and (vhs[j]==u',' or vhs[j]==u';'):
vhst.append(vhs[:j])
vhs=vhs[j+1:]
j=0
j+=1
vhst.append(vhs)
else:
vhst=[vhs]
for vhs in vhst:
p=iskat (vhs,u'(')
zakrsk=u')'
if p==-1:
p=iskat (vhs,u'[')
zakrsk=u']'
if p!=-1:
p1=iskat (vhs,zakrsk)
if p1!=-1:
if p==0 or vhs[p-1]==u' ':
vhs=vhs[:p]+vhs[p1+1:]
else:
vhs1=vhs[:p]+vhs[p1+1:]
fltolk1=fltolk
for vihs in vihst:
dobavslov(slov,vhjaz,vihjaz,vhs1,vihs,0,fltolk1)
if fltolk1==1:
fltolk1=0
fltolk1=fltolk
for vihs in vihst:
dobavslov(slov,vhjaz,vihjaz,vhs,vihs,0,fltolk1)
if fltolk1==1:
fltolk1=0
return
def zagruzslov_viki(slov,vhjaz,vihjaz,fslov,friwi):
f=codecs.open(fslov,'rb',encoding='utf-8')
j=0
jt0={}
st0={}
fiwi=1
if friwi:
fiwi=2
vihs=u''
for s in f.readlines():
# wikipedia.output(u'%d\n' % ord(s[0]))
if ord(s[0]) == 65279:
s=s[1:]
# wikipedia.output(u'%d %d %s %s\n' % (i,j,n,s))
s=s.replace(u'\r',u'')
if s[len(s)-1]==u'\n':
s=s[:len(s)-1]
if s == u'':
if j>=1 and vihs!=u'':
k=0
while k<j:
dobavslov(slov,jt0[k],vihjaz,st0[k],vihs,fiwi,0)
k+=1
j=0
jt0={}
st0={}
vihs=u''
else:
i=iskat (s,u':')
if i!=-1:
if s[:i]==vihjaz:
vihs=s[i+1:]
else:
if vhjaz==u'' or vhjaz==s[:i]:
jt0[j]=s[:i]
st0[j]=s[i+1:]
j+=1
return
def zagruz_ns(ns,fi):
f=codecs.open(fi,'rb',encoding='utf-8')
for s in f.readlines():
if ord(s[0]) == 65279:
s=s[1:]
s=ubr_nk_prob(s.replace(u'\r',u''))
if s != u'':
ns[s]=1
return
def zagruzslov(slov,slovdop,nssvoj,vhjaz,vihjaz,fslov):
f=codecs.open(fslov,'rb',encoding='utf-8')
for s in f.readlines():
if ord(s[0]) == 65279:
s=s[1:]
s=s.replace(u'\r',u'')
if s[len(s)-1]==u'\n':
s=s[:len(s)-1]
p=iskat(s,u'#')
if p!=-1:
s=s[:p]
if len(s)<1:
continue
st0=s.split(u' ',1)
tip=st0[0]
s=st0[1]
if tip==u'w':
fi=ubr_nk_prob(s)
zagruzslov_viki(slov,vhjaz,vihjaz,fi,0)
elif tip==u'wr':
fi=ubr_nk_prob(s)
zagruzslov_viki(slov,vhjaz,vihjaz,fi,1)
elif tip[0]==u'r':
st=s.split(u' ',2)
ftdob=0
if len(tip)>=2 and tip[1]==u'd':
ftdob=1
if (vhjaz==u'' or st[0]==vhjaz) and st[1]==vihjaz:
zagruzslov_revo(slov,st[0],vihjaz,ubr_nk_prob(st[2]),ftdob)
elif tip==u's':
st=s.split(u' ',1)
if st[0]==vihjaz:
zagruz_ns(nssvoj,ubr_nk_prob(st[1]))
# elif tip==u'c':
# fi=ubr_nk_prob(s)
# zagruz_ns(nscom,fi)
# elif tip==u'z':
# fi=ubr_nk_prob(s)
# zagruz_ns(zapriz,fi)
else:
if not slovdop.has_key(tip):
slovdop[tip]={}
fi=ubr_nk_prob(s)
zagruz_ns(slovdop[tip],fi)
class Perev_stat:
def __init__(self):
self.flp = 0
self.nup = 0
self.nnp = 0
self.nvap = 0
self.snp = []
self.svap = []
def korr_uktbl(uktbl,us,un,i):
if uktbl==None:
return i
while i<len(uktbl) and uktbl[i]<=us:
uktbl[i]+=un-us
if i>0 and uktbl[i]<uktbl[i-1]:
uktbl[i]=uktbl[i-1]
i+=1
return i
class Dslov:
def __init__(self,ti,na,ok,ok1,ok2):
self.ti = ti
self.na = na
self.ok = ok
self.ok1 = ok1
self.ok2 = ok2
def perevod_korr_eo(s,uktbl):
prom=[]
slov=[]
spslov=[]
p=0
while 1:
p0=p
while p<len(s) and not s[p].isalpha():
p+=1
prom.append(s[p0:p])
if p>=len(s):
break
p1=p
while p<len(s) and (s[p].isalpha() or
(p<len(s)-1 and s[p]==u'-' and s[p+1].isalpha())):
p+=1
slov.append(s[p1:p])
spslov.append(p1)
dslov=[]
for sl in slov:
# sl=sl.lower()
ti=0
ok2=u''
ok1=u''
if RBRm(sl,[u'la',u'kaj',u'tra',u'pro',u'do',u'na']):
na=sl
ok=u''
else:
na=sl[:len(sl)-1]
ok=sl[len(sl)-1]
if len(na)>=2 and RBR(ok,u'n'):
ok2=ok
ok=u''
sl=na
if RBRm(sl,[u'si',u'li',u'ĝi',u'ŝi']):
ti=110
na=sl
ok=u''
elif RBRm(sl,[u'ili',u'oni']):
ti=130
na=sl
ok=u''
else:
na=sl[:len(sl)-1]
ok=sl[len(sl)-1]
if len(na)>=2 and RBR(ok,u'j'):
ok1=ok
ok=u''
sl=na
na=sl[:len(sl)-1]
ok=sl[len(sl)-1]
if RBRm(sl,[u'iu',u'kiu',u'tiu',u'ĉiu',u'neniu']):
ti=210
else:
if RBR(ok,u'o'):
ti=100
elif RBR(ok,u'a'):
ti=200
elif ok1==u'' and RBR(ok,u'e'):
ti=300
elif ok1==u'' and ok2==u'' and RBRm(ok,[u'i',u'u']):
ti=400
elif (ok1==u'' and ok2==u'' and RBR(ok,u's') and
len(sl)>=4 and
RBRm(sl[len(sl)-2],[u'i',u'a',u'o',u'u'])):
na=sl[:len(sl)-2]
ok=sl[len(sl)-2:]
ti=400
else:
ti=0
if RBR(ok1,u'j'):
ti+=20
dslov.append(Dslov(ti,na,ok,ok1,ok2))
vivod(u' %s ti=%d\n'%(na+ok+ok1+ok2,ti))
i=len(dslov)-1
while i>=0:
if dslov[i].ti==400 and dslov[i].ok==u'is' and (i<1 or
dslov[i-1].ti==400 or
(dslov[i-1].ti==300 and (i<2 or dslov[i-2].ti==400)) ):
dslov[i].ti=200
dslov[i].na+=u'it'
dslov[i].ok=u'a'
i-=1
i=len(dslov)-1
while i>=0:
ti0=dslov[i].ti
# vivod(u' i=%d ti0=%d\n'%(i,ti0))
j=i-1
if ti0>=100 and ti0<200:
while j>=0:
if not (dslov[j].ti>=100 and dslov[j].ti<300):
break
if dslov[j].ok2!=u'':
break
if prom[j+1]!=u' ' and prom[j+1]!=u'\n':
break
vivod(u' j=%d ti=%d\n'%(j,dslov[j].ti))
if dslov[j].ti>=100 and dslov[j].ti<200:
dslov[j].ti+=100
dslov[j].ok=u'a'
if dslov[j].ti==210 or dslov[j].ti==230:
dslov[j].ti==200
if ti0>=120 and dslov[j].ti>=200 and dslov[j].ti<220:
dslov[j].ti+=20
dslov[j].ok1=u'j'
elif ti0<120 and dslov[j].ti>=220 and dslov[j].ti<240:
dslov[j].ti-=20
dslov[j].ok1=u''
j-=1
i=j
rez=u''
iuktbl=0
i=0
while i<len(dslov):
rez+=prom[i]
iuktbl=korr_uktbl(uktbl,spslov[i],len(rez),iuktbl)
ds=dslov[i]
rez+=ds.na+ds.ok+ds.ok1+ds.ok2
i+=1
rez+=prom[i]
korr_uktbl(uktbl,len(s),len(rez),iuktbl)
return rez
neperevt=[u'<br',u'<tr>',u'</tr>',u'<td>',u'</td>',u'<font',u'</font>',
u'<span',u'</span>',u'<b>',u'</b>',u'<i>',u'</i>',u'<s>',u'</s>',
u'<sub',u'</sub>',u'<sup',u'</sup>',u'<gallery>',u'</gallery>',
u'<center>',u'</center>',u'<blockquote>',u'</blockquote>',
u'<div',u'</div>',u'<small>',u'</small>']
neperevat=[u'width= ',u'colspan= ',u'border= ',u'style= ',u'align= ',u'id= ',
u'bgcolor= ',u'color= ',u'clear= ',u'valign= ',u'cellpadding= ',
u'cellspacing= ',
u'http://',
u'width=',u'colspan=',u'border=',u'style=',u'align=',u'id=',
u'bgcolor=',u'color=',u'clear=',u'valign=',u'cellpadding=',
u'cellspacing=',]
def perevod_stat(slov,vhjaz,vihjaz,s,uktbl=None):
pes=Perev_stat()
maxdl=48
while u' ' in s:
s = s.replace(u' ',u' ')
if len(s)<1:
return (u'',pes)
rez=u''
iuktbl=0
i=0
while i<len(s):
while i<len(s) and s[i]==u' ':
rez+=s[i]
i+=1
if i>=len(s):
break
fl=1
while fl:
fl=0
if i+3<len(s) and s[i]==u'&':
j=iskat(s[i:i+10],u';')
if j!=-1 and ( (s[i+1]==u'#' and s[i+2:i+j].isdigit()) or
s[i+1:i+j].isalpha() ):
np=s[i:i+j+1]
rez+=np
i+=len(np)
fl=1
break
for np in neperevt:
if s[i:i+len(np)].lower()==np:
rez+=np
i+=len(np)
fl=1
break
for np in neperevat:
if s[i:i+len(np)].lower()==np:
dl=len(np)
if i+dl<len(s) and s[i+dl]==u'"':
j=iskat(s[i+len(np)+1:],u'"')
if j!=-1:
dl+=2+j
else:
# j=iskat(s[i+dl:],u' ')
# if j==-1:
# j=iskat(s[i+dl:],u'\n')
(j,jj)=iskats_mn(s[i+dl:],0,[u' ',u'\n',u'|'])
if j!=-1:
dl+=j
iuktbl=korr_uktbl(uktbl,i,len(rez),iuktbl)
rez+=s[i:i+dl]
i+=dl
fl=1
break
if i>=len(s):
break
dl=maxdl
if i+dl>len(s):
dl=len(s)-i
while dl>0:
if dl>0 and i+dl<len(s) and s[i+dl].isalnum():
dl-=1
continue
tu=s[i:i+dl]
tu0=tu[0].upper()
flmb=(tu0!=tu[0])
tu=tu0+tu[1:]
if slov.has_key((vhjaz,tu)):
pes.flp=1
tp=slov[(vhjaz,tu)].tp
npe=len(tp)
if npe==1:
pe=tp[0]
if pe==u'_':
if i+dl<len(s) and s[i+dl]==u' ' and (s[i].islower() or
i<2 or (s[i-1] in [u'.',u'\n']) or
(s[i-2] in [u'.',u'\n'])):
pe=u''
if (i+dl+1<len(s) and s[i].isupper() and
s[i+dl+1].islower()):
s=s[:i+dl+1]+s[i+dl+1].upper()+s[i+dl+2:]
dl+=1
else:
pe=s[i:i+dl]
elif flmb:
pe=pe[0].lower()+pe[1:]
pes.nup+=1
else:
pe=u'('
k=0
while k<npe:
pe1=tp[k]
if flmb:
pe1=pe1[0].lower()+pe1[1:]
pe+=pe1
k+=1
if k<npe:
pe+=u', '
pe+=u')'
pes.nvap+=1
pes.svap.append(tu)
iuktbl=korr_uktbl(uktbl,i,len(rez),iuktbl)
rez+=pe
break
dl-=1
if dl>0:
i+=dl
else:
j=1
while i+j<len(s) and s[i+j-1].isalnum() and s[i+j].isalnum():
j+=1
iuktbl=korr_uktbl(uktbl,i,len(rez),iuktbl)
if s[i].isalnum() and not s[i:i+j].isdigit():
rez+=u'_'+s[i:i+j]+u'_'
pes.nnp+=1
pes.snp.append(perv_upper(s[i:i+j]))
else:
rez+=s[i:i+j]
i+=j
korr_uktbl(uktbl,len(s),len(rez),iuktbl)
if vihjaz==u'eo':
rez=perevod_korr_eo(rez,uktbl)
return (rez,pes)
def perevod(slov,vhjaz,vihjaz,s,uktbl=None):
(t,stat)=perevod_stat(slov,vhjaz,vihjaz,s,uktbl)
return (t,stat.flp)
def perevod_iwi_spis(slov,vhjaz,vihjaz,s):
s = s.replace(u'_',u' ')
while u' ' in s:
s = s.replace(u' ',u' ')
s=ubr_nk_prob(s)
if len(s)<1:
return []
tu=s
tu0=tu[0].upper()
flmb=(tu0!=tu[0])
tu=tu0+tu[1:]
if not slov.has_key((vhjaz,tu)):
return []
return slov[(vhjaz,tu)].iwi
def perevod_iwi(slov,vhjaz,vihjaz,s):
iwi=perevod_iwi_spis(slov,vhjaz,vihjaz,s)
npe=len(iwi)
if npe<1:
return u''
if npe==1:
pe=iwi[0]
# if flmb:
# pe=pe[0].lower()+pe[1:]
pe=u'[['+pe+u']]'
else:
pe=u''
k=0
while k<npe:
pe1=iwi[k]
# if flmb:
# pe1=pe1[0].lower()+pe1[1:]
pe+=u'[['+pe1+u']]'
k+=1
if k<npe:
pe+=u', '
return pe
def perev_uch_nezsl(osnp,osvap,stat):
for t in stat.snp:
if osnp.has_key(t):
osnp[t]+=1
else:
osnp[t]=1
for t in stat.svap:
if osvap.has_key(t):
osvap[t]+=1
else:
osvap[t]=1
def perev_uch_nezsl_rt(osnp,osvap,stat):
for t in stat.snp:
osnp[t]=1
for t in stat.svap:
osvap[t]=1
def perev_pech_nezsl(slov,vhjaz,vihjaz,osnp,osvap,fnesl,kolotch):
ssnp=[]
for t, n in osnp.iteritems():
ssnp.append((n,t))
ssnp.sort(reverse=True)
i=0
for n, t in ssnp:
fnesl.write(u'%d %s\n'%(n,t))
fnesl.flush()
i+=1
if i>=kolotch:
break
ssvap=[]
for t, n in osvap.iteritems():
ssvap.append((n,t))
fnesl.write(u'\n')
fnesl.flush()
ssvap.sort(reverse=True)
i=0
for n, t in ssvap:
fnesl.write(u'%d %s\n'%(n,t))
fnesl.flush()
i+=1
if i>=kolotch:
break
fnesl.write(u'\n')
fnesl.flush()
i=0
for n, t in ssnp:
fnesl.write(u'%s\n'%t)
fnesl.flush()
i+=1
if i>=kolotch:
break
fnesl.write(u'\n')
fnesl.flush()
i=0
for n, t in ssvap:
(t2,stat2)=perevod_stat(slov,vhjaz,vihjaz,t)
if len(t2)>=1 and t2[0]==u'(':
t2=t2[1:]
if len(t2)>=1 and t2[len(t2)-1]==u')':
t2=t2[:len(t2)-1]
fnesl.write(u'! %s : %s\n'%(t,t2))
fnesl.flush()
i+=1
if i>=kolotch:
break
def main(slov,vhjaz,vihjaz,fvh,fvih):
kolotch=500
f0=codecs.open(fvh,'rb',encoding='utf-8')
f1=codecs.open(fvih, 'w', 'utf-8')
osnp={}
osvap={}
for s in f0.readlines():
# wikipedia.output(u'%d\n' % ord(s[0]))
if ord(s[0]) == 65279:
s=s[1:]
# wikipedia.output(u'%d %d %s %s\n' % (i,j,n,s))
s=s.replace(u'\r',u'').replace(u'_',u' ')
# if s[len(s)-1]!=u'\n':
# s=s+u'\n'
# (rez,fl_perev)=perevod(slov,vhjaz,vihjaz,s)
(rez,stat)=perevod_stat(slov,vhjaz,vihjaz,s)
f1.write(rez)
f1.flush()
perev_uch_nezsl(osnp,osvap,stat)
perev_pech_nezsl(slov,vhjaz,vihjaz,osnp,osvap,otch,kolotch)
if __name__ == "__main__":
try:
vhjaz = wikipedia.argHandler(sys.argv[1], 'perevod')
vihjaz = wikipedia.argHandler(sys.argv[2], 'perevod')
fslov = wikipedia.argHandler(sys.argv[3], 'perevod')
fvh = wikipedia.argHandler(sys.argv[4], 'perevod')
fvih = wikipedia.argHandler(sys.argv[5], 'perevod')
fotch = wikipedia.argHandler(sys.argv[6], 'perevod')
otch = codecs.open(fotch, 'w', 'utf-8')
#otch2 = codecs.open(fotch2, 'w', 'utf-8')
slov={}
nssvoj={}
slovdop={}
zagruzslov(slov,slovdop,nssvoj,vhjaz,vihjaz,fslov)
main(slov,vhjaz,vihjaz,fvh,fvih)
finally:
wikipedia.stopme()
* upload1.py
# -*- coding: utf-8 -*-
"""
Script to upload images to wikipedia.
Arguments:
-keep Keep the filename as is
If any other arguments are given, the first is the URL or filename
to upload, and the rest is a proposed description to go with the
upload. If none of these are given, the user is asked for the
file or URL to upload. The bot will then upload the image to the wiki.
The script will ask for the location of an image, if not given as a parameter,
and for a description.
"""
#
# (C) Rob W.W. Hooft, Andre Engels 2003-2004
#
# Distribute under the terms of the PSF license.
#
__version__='$Id: upload.py,v 1.31 2005/08/13 15:45:35 wikipedian Exp $'
import os, sys, re
import urllib, httplib
import wikipedia, config, mediawiki_messages
def iskats(t,i,s):
lt=len(t)
ls=len(s)
while i<=lt-ls:
if t[i:i+ls]==s:
return i
i+=1
return -1
def post_multipart(host, selector, fields, files, cookies):
"""
Post fields and files to an http host as multipart/form-data.
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be uploaded as files
Return the server's response page.
"""
content_type, body = encode_multipart_formdata(fields, files)
conn = httplib.HTTPConnection(host)
conn.putrequest('POST', selector)
conn.putheader('content-type', content_type)
conn.putheader('content-length', str(len(body)))
conn.putheader("User-agent", "RobHooftWikiRobot/1.0")
conn.putheader('Host', host)
if cookies:
conn.putheader('Cookie',cookies)
conn.endheaders()
conn.send(body)
response = conn.getresponse()
returned_html = response.read()
conn.close()
return response, returned_html
def encode_multipart_formdata(fields, files):
"""
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be uploaded as files
Return (content_type, body) ready for httplib.HTTP instance
"""
BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
CRLF = '\r\n'
L = []
for (key, value) in fields:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append('')
L.append(value)
for (key, filename, value) in files:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
L.append('Content-Type: %s' % get_content_type(filename))
L.append('')
L.append(value)
L.append('--' + BOUNDARY + '--')
L.append('')
body = CRLF.join(L)
content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
return content_type, body
def get_content_type(filename):
import mimetypes
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
class UploadRobot:
def __init__(self, url, description = u'', keepFilename = False, targetSite = None, urlEncoding = None,otchup=None):
self.url = url
self.otchup = otchup
self.urlEncoding = urlEncoding
self.description = description
self.keepFilename = keepFilename
if config.upload_to_commons:
self.targetSite = targetSite or wikipedia.getSite('commons', 'commons')
else:
self.targetSite = targetSite or wikipedia.getSite()
self.targetSite.forceLogin()
def urlOK(self):
'''
Returns true iff the URL references an online site or an
existing local file.
'''
return self.url != '' and ('://' in self.url or os.path.exists(self.url))
def upload_image(self, debug=False):
"""Gets the image at URL self.url, and uploads it to the target wiki.
Returns (filename,tekst oshibki)
"""
tosh=u''
# Get file contents
if '://' in self.url:
uo = wikipedia.MyURLopener()
try:
file = uo.open(self.url)
except IOError:
return (u'',u'wikipedia.MyURLopener open IOError')
else:
# Opening local files with MyURLopener would be possible, but we
# don't do it because it only accepts ASCII characters in the
# filename.
file = open(self.url)
wikipedia.output(u'Reading file %s' % self.url)
try:
contents = file.read()
except IOError:
return (u'',u'read IOError')
file.close()
if len(contents)<10 or contents.startswith('<html'):
tosh="Couldn't download the image."
print tosh
self.otchup.write(u'======oshibka======\n%s\n==================\n' % tosh)
self.otchup.flush()
return (u'',tosh)
# Isolate the pure name
filename = self.url
if '/' in filename:
filename = filename.split('/')[-1]
if '\\' in filename:
filename = filename.split('\\')[-1]
if self.urlEncoding:
filename = urllib.unquote(filename)
filename = filename.decode(self.urlEncoding)
if not self.keepFilename:
wikipedia.output(u"The filename on the target wiki will default to: %s" % filename)
# ask newfn until it's valid
ok = False
# FIXME: these 2 belong somewhere else, presumably in family
forbidden = '/' # to be extended
allowed_formats = (u'jpg', u'jpeg', u'png', u'gif', u'svg', u'ogg')
newfn = filename
# while not ok:
# ok = True
# newfn = wikipedia.input(u'Enter a better name, or press enter to accept:')
# if newfn == "":
# newfn = filename
# ext = os.path.splitext(newfn)[1].lower().strip('.')
# for c in forbidden:
# if c in newfn:
# print "Invalid character: %s. Please try again" % c
# ok = False
# if ext not in allowed_formats and ok:
# ans = wikipedia.input(u"File format is not %s but %s. Continue [y/N]? " % (allowed_formats, ext))
# if not ans.lower().startswith('y'):
# ok = False
# if not ok:
# newfn = wikipedia.input(u'Enter a better name, or press enter to accept:')
if newfn != '':
filename = newfn
# MediaWiki doesn't allow spaces in the file name.
# Replace them here to avoid an extra confirmation form
filename = filename.replace(' ', '_')
# Convert the filename (currently Unicode) to the encoding used on the
# target wiki
encodedFilename = filename.encode(self.targetSite.encoding())
# A proper description for the submission.
wikipedia.output(u"The suggested description is:")
wikipedia.output(self.description)
# choice = wikipedia.inputChoice(u'Do you want to change this description?', ['Yes', 'No'], ['y', 'N'], 'n')
# if choice not in ['n', 'N']:
# newDescription = wikipedia.ui.editText(self.description)
# # if user didn't press Cancel:
# if newDescription:
# self.description = newDescription
formdata = {}
formdata["wpUploadDescription"] = self.description
# if self.targetSite.version() >= '1.5':
# formdata["wpUploadCopyStatus"] = wikipedia.input(u"Copyright status: ")
# formdata["wpUploadSource"] = wikipedia.input(u"Source of image: ")
formdata["wpUploadAffirm"] = "1"
formdata["wpUpload"] = "upload bestand"
formdata["wpIgnoreWarning"] = "1"
# try to encode the strings to the encoding used by the target site.
# if that's not possible (e.g. because there are non-Latin-1 characters and
# the home Wikipedia uses Latin-1), convert all non-ASCII characters to
# HTML entities.
for key in formdata:
assert isinstance(key, basestring), "ERROR: %s is not a string but %s" % (key, type(key))
try:
formdata[key] = formdata[key].encode(self.targetSite.encoding())
except (UnicodeEncodeError, UnicodeDecodeError):
formdata[key] = wikipedia.UnicodeToAsciiHtml(formdata[key]).encode(self.targetSite.encoding())
# don't upload if we're in debug mode
if not debug:
wikipedia.output(u'Uploading file to %s...' % self.targetSite)
response, returned_html = post_multipart(self.targetSite.hostname(),
self.targetSite.upload_address(),
formdata.items(),
(('wpUploadFile', encodedFilename, contents),),
cookies = self.targetSite.cookies()
)
returned_html = returned_html.decode(self.targetSite.encoding())
# Do we know how the "success!" HTML page should look like?
# ATTENTION: if you changed your Wikimedia Commons account not to show
# an English interface, this detection will fail!
#success_msg = mediawiki_messages.get('successfulupload', site = self.targetSite)
#success_msgR = re.compile(re.escape(success_msg))
#if success_msgR.search(returned_html):
# wikipedia.output(u"Upload successful.")
# dump the HTML page
# wikipedia.output(u'%s\n\n' % returned_html)
# wikipedia.output(u'%i %s' % (response.status, response.reason))
self.otchup.write(u'%s\n============\n%s\n==================\n' % (filename,returned_html))
self.otchup.flush()
fosh=0
tosh=returned_html
if len(tosh)>20:
fosh=1
tnosh='<span class=\'error\'>'
p1=iskats(tosh,0,tnosh)
if p1!=-1:
p1+=len(tnosh)
p2=iskats(tosh,p1,'</span>')
if p2!=-1:
tosh=tosh[p1:p2]
if not (response.status in [200, 302]):
fosh=1
if not fosh:
tosh=u''
if fosh and len(tosh)<2:
tosh=u'!!!!oshibka!!!!'
tosh=tosh[:300].replace('\n',' ').replace('\r',' ')
if fosh:
self.otchup.write(u'======oshibka======\n%s\n' % tosh)
self.otchup.flush()
else:
wikipedia.output(u"Upload successful.")
self.otchup.write(u'<- ++\n')
self.otchup.flush()
self.otchup.write(u'============================\n')
self.otchup.flush()
return (filename,tosh)
def run(self):
# while not self.urlOK():
# if not self.url:
# wikipedia.output(u'No input filename given')
# else:
# wikipedia.output(u'Invalid input filename given. Try again.')
# self.url = wikipedia.input(u'File or URL where image is now:')
return self.upload_image()
def main(args):
url = u''
description = []
keepFilename = False
for arg in args:
arg = wikipedia.argHandler(arg, 'upload')
if arg:
if arg.startswith('-keep'):
keepFilename = True
elif url == u'':
url = arg
else:
description.append(arg)
description = u' '.join(description)
bot = UploadRobot(url, description, keepFilename)
bot.run()
if __name__ == "__main__":
try:
main(sys.argv[1:])
finally:
wikipedia.stopme()
* com-izobr.py
# -*- coding: UTF-8 -*-
__version__='$Id:'
import wikipedia, pagegenerators
import re, sys
import codecs
import perevod,imagetransfer1
def iskat(t,s):
lt=len(t)
ls=len(s)
i=0
while i<=lt-ls:
if t[i:i+ls]==s:
return i
i+=1
return -1
def iskats(t,i,s):
lt=len(t)
ls=len(s)
while i<=lt-ls:
if t[i:i+ls]==s:
return i
i+=1
return -1
def iskato(t,i,s):
ls=len(s)
while i>=0:
if t[i:i+ls]==s:
return i
i-=1
return -1
def vivod(b):
wikipedia.output(b)
otch.write(b)
otch.flush()
def ubr_nk_prob(t):
if len(t)<1:
return t
while (t[0]==u' ') or (t[0]==u'\n') or (t[0]==u'\t') or (t[0]==u'\r'):
t=t[1:]
if len(t)<1:
return t
while (t[len(t)-1]==u' ') or (t[len(t)-1]==u'\n') or (t[len(t)-1]==u'\t') or (t[len(t)-1]==u'\r'):
t=t[:len(t)-1]
if len(t)<1:
return t
return t
otch.write(u'\n\n\n')
otch.flush()
zapis_fimen(fimen,nomerf,pl)
return
def main():
# mysite = wikipedia.getSite()
sico=wikipedia.getSite(code = u'commons',fam = u'commons')
global otch
otch = codecs.open(u'%s' % filename, 'w', 'utf-8')
for page in sico.allpages(start = u'', namespace = 6):
vivod(page.title()+u'\n')
filename = wikipedia.argHandler(sys.argv[1], 'cht_cat')
for arg in sys.argv[2:]:
arg = wikipedia.argHandler(arg, 'ivsen')
if arg:
if arg=="-log":
import logger
sys.stdout = logger.Logger(sys.stdout, filename = 'com-izobr.log')
try:
main()
except:
wikipedia.stopme()
raise
else:
wikipedia.stopme()
* com-kateg.py
# -*- coding: UTF-8 -*-
__version__='$Id:'
import wikipedia, pagegenerators
import re, sys
import codecs
import perevod,imagetransfer1
def iskat(t,s):
lt=len(t)
ls=len(s)
i=0
while i<=lt-ls:
if t[i:i+ls]==s:
return i
i+=1
return -1
def iskats(t,i,s):
lt=len(t)
ls=len(s)
while i<=lt-ls:
if t[i:i+ls]==s:
return i
i+=1
return -1
def iskato(t,i,s):
ls=len(s)
while i>=0:
if t[i:i+ls]==s:
return i
i-=1
return -1
def vivod(b):
wikipedia.output(b)
otch.write(b)
otch.flush()
def ubr_nk_prob(t):
if len(t)<1:
return t
while (t[0]==u' ') or (t[0]==u'\n') or (t[0]==u'\t') or (t[0]==u'\r'):
t=t[1:]
if len(t)<1:
return t
while (t[len(t)-1]==u' ') or (t[len(t)-1]==u'\n') or (t[len(t)-1]==u'\t') or (t[len(t)-1]==u'\r'):
t=t[:len(t)-1]
if len(t)<1:
return t
return t
otch.write(u'\n\n\n')
otch.flush()
zapis_fimen(fimen,nomerf,pl)
return
def main():
# mysite = wikipedia.getSite()
sico=wikipedia.getSite(code = u'commons',fam = u'commons')
global otch
otch = codecs.open(u'%s' % filename, 'w', 'utf-8')
for page in sico.allpages(start = u'', namespace = 14):
vivod(page.title()+u'\n')
filename = wikipedia.argHandler(sys.argv[1], 'cht_cat')
for arg in sys.argv[2:]:
arg = wikipedia.argHandler(arg, 'ivsen')
if arg:
if arg=="-log":
import logger
sys.stdout = logger.Logger(sys.stdout, filename = 'com-izobr.log')
try:
main()
except:
wikipedia.stopme()
raise
else:
wikipedia.stopme()
* slov_iz_xml.py
# -*- coding: UTF-8 -*-
__version__='$Id:'
import wikipedia
import re, sys
import codecs
import xmlreader
from bib_tekst import *
def ubrkoment (text):
# Ignore tekst within nowiki tags and HTML comments
nowikiOrHtmlCommentR = re.compile(r'<nowiki>.*?|<!--.*?-->', re.IGNORECASE | re.DOTALL)
match = nowikiOrHtmlCommentR.search(text)
while match:
text = text[:match.start()] + text[match.end():]
match = nowikiOrHtmlCommentR.search(text)
return text
def korr_im (jaz,t,pref):
for p in pref:
if t.startswith(p):
t=t[len(p):]
break
t=perv_upper(t)
if u':' in t:
if ( (not t.startswith(u'Category:')) and
(not t.startswith(u'Image:')) ):
katprefi=mysite.family.category_namespace(jaz)
if not t.startswith(katprefi+u':'):
izprefi=mysite.family.image_namespace(jaz)
if not t.startswith(izprefi+u':'):
return u''
return t
def main(vhjaz,fvh,fvih,fipref):
tzfl=0
tzst={}
tzno={}
pref=[]
if fipref!=u'':
fpref=codecs.open(fipref,'rb',encoding='utf-8')
for s in fpref.readlines():
if ord(s[0]) == 65279:
s=s[1:]
s=s.replace(u'\r',u'')
if s[len(s)-1]==u'\n':
s=s[:len(s)-1]
pref.append(s)
fpref.close()
n=u''
# f0=codecs.open(fvh,'rb',encoding='utf-8')
f1=codecs.open(fvih, 'w', 'utf-8')
insite=wikipedia.getSite(vhjaz,fam = u'wikipedia')
tblredir = {}
# open xml dump and read page titles out of it
dump = xmlreader.XmlDump(fvh)
redirR = wikipedia.getSite().redirectRegex()
readPagesCount = 0
for entry in dump.parse():
readPagesCount += 1
# always print status message after 10000 pages
if readPagesCount % 10000 == 0:
print '%i pages read...' % readPagesCount
m = redirR.search(entry.text)
if m:
"""
target = m.group(1)
# There might be redirects to another wiki. Ignore these.
for code in wikipedia.getSite().family.langs.keys():
if target.startswith('%s:' % code) or target.startswith(':%s:' % code):
wikipedia.output(u'NOTE: Ignoring %s which is a redirect to %s:' % (entry.title, code))
target = None
break
# if the redirect does not link to another wiki
if target:
target = target.replace(' ', '_')
# remove leading and trailing whitespace
target = target.strip()
# capitalize the first letter
if not wikipedia.getSite().nocapitalize:
target = target[0].upper() + target[1:]
if '#' in target:
target = target[:target.index('#')]
if '|' in target:
wikipedia.output(u'HINT: %s is a redirect with a pipelink.' % entry.title)
target = target[:target.index('|')]
tblredir[entry.title] = target
"""
pass
else:
te=ubrkoment(entry.text)
fperv=1
interwikiR = re.compile(r'\[\[([a-z\-]+)\s?:([^\[\]\n]*)\]\]')
for lang, pagetitle in interwikiR.findall(te):
# Check if it really is in fact an interwiki link to a known
# language, or if it's e.g. a category tag or an internal link
if lang in insite.family.langs:
if '|' in pagetitle:
# ignore text after the pipe
pagetitle = pagetitle[:pagetitle.index('|')]
b1=korr_im (lang, pagetitle,pref)
if b1==u'':
continue
if fperv:
# b=u'%s:%s\n' % (vhjaz,entry.title)
b0=korr_im(vhjaz,entry.title,pref)
if b0==u'':
break
b=u'%s:%s\n' % (vhjaz,b0)
f1.write(b)
fperv=0
# b=u'%s:%s\n' % (lang, pagetitle)
b=u'%s:%s\n' % (lang, b1)
f1.write(b)
if fperv==0:
f1.write(u'\n')
f1.flush()
vhjaz = wikipedia.argHandler(sys.argv[1], 'slov_iz_xml')
fvh = wikipedia.argHandler(sys.argv[2], 'slov_iz_xml')
fvih = wikipedia.argHandler(sys.argv[3], 'slov_iz_xml')
fpref=u''
if len(sys.argv)>=5:
fpref = wikipedia.argHandler(sys.argv[4], 'slov_iz_xml')
#fotch = wikipedia.argHandler(sys.argv[4], 'slov_iz_xml')
#mysite = wikipedia.getSite()
#otch = codecs.open(fotch, 'w', 'utf-8')
mysite = wikipedia.getSite()
try:
main(vhjaz,fvh,fvih,fpref)
except:
wikipedia.stopme()
raise
else:
wikipedia.stopme()
</nowiki>* svoj-iz-ka.py
# -*- coding: UTF-8 -*-
__version__='$Id:'
import wikipedia, pagegenerators
import re, sys
import codecs
import perevod,imagetransfer1
def iskat(t,s):
lt=len(t)
ls=len(s)
i=0
while i<=lt-ls:
if t[i:i+ls]==s:
return i
i+=1
return -1
def iskats(t,i,s):
lt=len(t)
ls=len(s)
while i<=lt-ls:
if t[i:i+ls]==s:
return i
i+=1
return -1
def iskato(t,i,s):
ls=len(s)
while i>=0:
if t[i:i+ls]==s:
return i
i-=1
return -1
def vivod(b):
wikipedia.output(b)
otch.write(b)
otch.flush()
def ubr_nk_prob(t):
if len(t)<1:
return t
while (t[0]==u' ') or (t[0]==u'\n') or (t[0]==u'\t') or (t[0]==u'\r'):
t=t[1:]
if len(t)<1:
return t
while (t[len(t)-1]==u' ') or (t[len(t)-1]==u'\n') or (t[len(t)-1]==u'\t') or (t[len(t)-1]==u'\r'):
t=t[:len(t)-1]
if len(t)<1:
return t
return t
otch.write(u'\n\n\n')
otch.flush()
zapis_fimen(fimen,nomerf,pl)
return
def main():
mysite = wikipedia.getSite()
# sico=wikipedia.getSite(code = u'commons',fam = u'commons')
global otch
otch = codecs.open(u'%s' % filename, 'w', 'utf-8')
for page in mysite.allpages(start = u'', namespace = 6):
vivod(page.title()+u'\n')
for page in mysite.allpages(start = u'', namespace = 14):
vivod(page.title()+u'\n')
filename = wikipedia.argHandler(sys.argv[1], 'cht_cat')
for arg in sys.argv[2:]:
arg = wikipedia.argHandler(arg, 'ivsen')
if arg:
if arg=="-log":
import logger
sys.stdout = logger.Logger(sys.stdout, filename = 'com-izobr.log')
try:
main()
except:
wikipedia.stopme()
raise
else:
wikipedia.stopme()
* bib_kateg.py
# -*- coding: UTF-8 -*-
__version__='$Id:'
import wikipedia
import re, sys, os
import codecs
import perevod
from bib_tekst import *
from get_buf import *
def vivod(otch,b):
wikipedia.output(b)
otch.write(b)
otch.flush()
class Kateg:
def __init__(self, jaz,n,f=0):
if f:
p=iskat(n,u':')
if p!=-1:
n=n[p+1:]
self.jaz = jaz
self.n = perv_upper(ubr_nk_prob(n))
def perev_kateg(slov,nssvoj,si0,kato,fint,otch):
jaz0=si0.lang
nt_kat_sp=[]
glb=0
tbl_pl={}
kato1=kato
while 1:
katprefi=si0.family.category_namespace(jaz0)
for ka in kato1:
jaz=ka.jaz
issite=wikipedia.getSite(jaz,fam = si0.family)
iskatprefi=issite.family.category_namespace(jaz)
t=iskatprefi+u':'+ka.n
vivod(otch,u' prob kat %s:%s\n'%(jaz,t))
t0s=perevod.perevod_iwi_spis(slov,jaz,jaz0,t)
t1s=[]
for t0 in t0s:
if nssvoj.has_key(t0):
vivod(otch,u' +kat %s\n'%t0)
t1s.append(t0)
if len(t1s)<1:
t2s=perevod.perevod_iwi_spis(slov,jaz,jaz0,ka.n)
if len(t2s)>0:
t2=t2s[0]
vivod(otch,u' +perev sta %s\n'%t2)
t2=katprefi+u':'+t2
if nssvoj.has_key(t2):
vivod(otch,u' +perev->kat\n')
t1=t2
t1s=[t1]
elif nssvoj.has_key(t2+u'j'):
vivod(otch,u' +perev->kat +\'j\'\n')
t1=t2+u'j'
t1s=[t1]
for t1 in t1s:
vivod(otch,u' [[%s]]\n'%t1)
if not (t1 in nt_kat_sp):
nt_kat_sp.append(t1)
if (not fint) or len(nt_kat_sp)>0 or glb>=10:
break
nkpshpl=[]
for ka in kato1:
jaz=ka.jaz
issite=wikipedia.getSite(jaz,fam = si0.family)
iskatprefi=issite.family.category_namespace(jaz)
t=iskatprefi+u':'+ka.n
kapl=wikipedia.Page(issite,t)
nkpshpl.append(kapl)
sozd_buf_tbl(tbl_pl,nkpshpl,otch)
kato2=[]
for kapl in nkpshpl:
try:
kapl=get_tbl(tbl_pl,kapl)
kapl.get()
b = u' + kateg:%s\n' % kapl.title()
vivod(otch,b)
# otch2.write(u'%s\n%s\n========\n'%(kapl.title(),kapl.get()))
# otch2.flush()
plikat = kapl.categories()
for pka in plikat:
kato2.append(Kateg(kapl.site().lang,pka.title(),1))
vivod(otch,u' + %s\n'%pka.title())
except wikipedia.NoPage:
b = u' - kateg:%s\n' % kapl.title()
vivod(otch,b)
except wikipedia.IsRedirectPage:
b = u' + kateg:%s\n -> %s\n' % (kapl.title(),
kapl.getRedirectTarget())
vivod(otch,b)
kato2.append(Kateg(kapl.site().lang,kapl.getRedirectTarget(),1))
kato1=kato2
glb+=1
vivod(otch,u' * kateg:glb=%d\n' % glb)
return nt_kat_sp
def kateg_v_tekst(nt_kat_sp):
nt_kat=u''
for t in nt_kat_sp:
t= u'[[%s]]\n'%t
nt_kat+=t
return nt_kat
* bib_tekst.py
def RBR(a,b):
return a.lower()==b.lower()
def RBRm(a,b):
for b1 in b:
if a.lower()==b1.lower():
return 1
return 0
def perv_upper(t):
if len(t)<1:
return u''
return t[0].upper()+t[1:]
def perv_lower(t):
if len(t)<1:
return u''
return t[0].lower()+t[1:]
def perv_lower_s(s):
r=[]
for t in s:
r.append(perv_lower(t))
return r
def iskat(t,s):
lt=len(t)
ls=len(s)
i=0
while i<=lt-ls:
if t[i:i+ls]==s:
return i
i+=1
return -1
def iskats(t,i,s):
lt=len(t)
ls=len(s)
while i<=lt-ls:
if t[i:i+ls]==s:
return i
i+=1
return -1
def iskato(t,i,s):
ls=len(s)
while i>=0:
if t[i:i+ls]==s:
return i
i-=1
return -1
def iskats_mn(t,p0,ss):
i=p0
while i<len(t):
for j in range(len(ss)):
if t[i:].startswith(ss[j]):
return (i,j)
i+=1
return (-1,0)
def iskkonpodp(t,i,os,zs):
j=0
while i<len(t):
if t[i:].startswith(os):
j+=1
i+=len(os)
continue
elif t[i:].startswith(zs):
j-=1
if j==-1:
return i
i+=len(zs)
continue
i+=1
return -1
def ubr_nk_prob(t):
i=0
while i<len(t) and ((t[i]==u' ') or (t[i]==u'\n') or (t[i]==u'\t') or
(t[i]==u'\r')):
i+=1
j=len(t)-1
while j>i and ((t[j]==u' ') or (t[j]==u'\n') or (t[j]==u'\t') or
(t[j]==u'\r')):
j-=1
return t[i:j+1]
* get_buf.py
import wikipedia, pagegenerators
import re, sys, os
import codecs,xmlreader
def vivod(otch,b):
wikipedia.output(b)
otch.write(b)
otch.flush()
def getall_ex(si,s):
try:
wikipedia.getall(si,s)
except xmlreader.xml.sax._exceptions.SAXParseException:
pass
def get1(pl):
getall_ex(pl.site(), [pl])
def get_tbl(tbl,pl):
fa=pl.site().family.name
la=pl.site().lang
ti=pl.title()
if tbl.has_key((fa,la,ti)):
return tbl[(fa,la,ti)]
get1(pl)
return pl
def sozd_buf_tbl(tbl,nkp,otch):
f=[0 for i in range(len(nkp))]
for i in range(len(nkp)):
if f[i]:
continue
pl=nkp[i]
fa=pl.site().family.name
la=pl.site().lang
vivod(otch,u'sozd_buf_tbl %s %s %s\n'%(fa,la,pl.title()))
ind=(fa,la,pl.title())
if tbl.has_key(ind):
vivod(otch,u' <- uzxe\n')
continue
tbl[ind]=pl
t1=[pl]
j=i+1
while j<len(nkp):
if nkp[j].site().family.name==fa and nkp[j].site().lang==la:
vivod(otch,u'sozd_buf_tbl %s %s %s\n'%(fa,la,nkp[j].title()))
ind=(fa,la,nkp[j].title())
if not tbl.has_key(ind):
tbl[ind]=nkp[j]
t1.append(nkp[j])
else:
vivod(otch,u' <- uzxe\n')
f[j]=1
j+=1
getall_ex(pl.site(), t1)
vivod(otch,u'\n')