import pywikibot
from pywikibot import pagegenerators
#import pywikibot.data.wikidataquery as wdquery
from pywikibot import pagegenerators as pg
import codecs #used in logfiles, unicoded strings
import sys
import datetime
from datetime import datetime, date, time
replacedesc={'nl':['taxon','',]}
taxondescs={
'family of alga' :{'nl':u'taxon, familie van algen'},
'family of algae' :{'nl':u'taxon, familie van algachtigen'},
'family of arachnids' :{'nl':u'taxon, familie van spinachtigen'},
'family of birds' :{'nl':u'taxon, familie van vogels'},
'family of brachiopods' :{'nl':u'taxon, familie van armpotigen'},
'family of bryozoans' :{'nl':u'taxon, familie van mosdiertjes'},
'family of cnidarians' :{'nl':u'taxon, familie van neteldieren'},
'family of crustaceans' :{'nl':u'taxon, familie van kreeftachtigen'},
'family of echinoderms' :{'nl':u'taxon, familie van stekelhuidigen'},
'family of fishes' :{'nl':u'taxon, familie van vissen'},
'family of fungi' :{'nl':u'taxon, familie van schimmels'},
'family of gastrotrichs' :{'nl':u'taxon, familie van buikharigen'},
'family of insects' :{'nl':u'taxon, familie van insecten'},
'family of mammals' :{'nl':u'taxon, familie van zoogdieren'},
'family of molluscs' :{'nl':u'taxon, familie van weekdieren'},
'family of plants' :{'nl':u'taxon, familie van planten'},
'family of prokaryotes' :{'nl':u'taxon, familie van prokaryoten'},
'family of reptiles' :{'nl':u'taxon, familie van reptielen'},
'family of sea spiders' :{'nl':u'taxon, familie van zeespinnen'},
'family of waterbears' :{'nl':u'taxon, familie van beerdiertjes'},
'family of worms' :{'nl':u'taxon, familie van wormen'},
'genus of alga' :{'nl':u'taxon, geslacht van algen'},
'genus of algae' :{'nl':u'taxon, geslacht van algachtigen'},
'genus of amphibians' :{'nl':u'taxon, geslacht van amfibieën'},
'genus of arachnids' :{'nl':u'taxon, geslacht van spinachtigen'},
'genus of arthropods' :{'nl':u'taxon, geslacht van duizendpoten'},
'genus of birds' :{'nl':u'taxon, geslacht van vogels'},
'genus of brachiopods' :{'nl':u'taxon, geslacht van armpotigen'},
'genus of bryozoans' :{'nl':u'taxon, geslacht van mosdiertjes'},
'genus of cnidarian' :{'nl':u'taxon, geslacht van neteldieren'},
'genus of cnidarians' :{'nl':u'taxon, geslacht van neteldieren'},
'genus of crustaceans' :{'nl':u'taxon, geslacht van kreeftachtigen'},
'genus of echinoderms' :{'nl':u'taxon, geslacht van stekelhuidigen'},
'genus of fishes' :{'nl':u'taxon, geslacht van vissen'},
'genus of fungi' :{'nl':u'taxon, geslacht van schimmels'},
'genus of gastrotrichs' :{'nl':u'taxon, geslacht van buikharigen'},
'genus of insects' :{'nl':u'taxon, geslacht van insecten'},
'genus of mammals' :{'nl':u'taxon, geslacht van zoogdieren'},
'genus of molluscs' :{'nl':u'taxon, geslacht van weekdieren'},
'genus of myriapods' :{'nl':u'taxon, geslacht van duizendpotigen'},
'genus of plants' :{'nl':u'taxon, geslacht van planten'},
'genus of prokaryotes' :{'nl':u'taxon, geslacht van prokaryoten'},
'genus of reptiles' :{'nl':u'taxon, geslacht van reptielen'},
'genus of sea spiders' :{'nl':u'taxon, geslacht van zeespinnen'},
'genus of sponges' :{'nl':u'taxon, geslacht van sponsachtigen'},
'genus of trilobites' :{'nl':u'taxon, geslacht van drielobbigen'},
'genus of viruses' :{'nl':u'taxon, geslacht van virussen'},
'genus of waterbears' :{'nl':u'taxon, geslacht van beerdiertjes'},
'genus of worms' :{'nl':u'taxon, geslacht van wormen'},
'nothospecies of plant' :{'nl':u'taxon, nothospecies van planten'},
'order of alga' :{'nl':u'taxon, orde van algen'},
'order of algae' :{'nl':u'taxon, orde van algachtigen'},
'order of amphibians' :{'nl':u'taxon, orde van amfibieën'},
'order of arachnids' :{'nl':u'taxon, orde van spinachtigen'},
'order of birds' :{'nl':u'taxon, orde van vogels'},
'order of brachiopods' :{'nl':u'taxon, orde van armpotigen'},
'order of cnidarian' :{'nl':u'taxon, orde van neteldieren'},
'order of crustaceans' :{'nl':u'taxon, orde van kreeftachtigen'},
'order of fishes' :{'nl':u'taxon, orde van vissen'},
'order of fungi' :{'nl':u'taxon, orde van schimmels'},
'order of gastrotrichs' :{'nl':u'taxon, orde van buikharigen'},
'order of insects' :{'nl':u'taxon, orde van insecten'},
'order of mammals' :{'nl':u'taxon, orde van zoogdieren'},
'order of molluscs' :{'nl':u'taxon, orde van weekdieren'},
'order of plants' :{'nl':u'taxon, orde van planten'},
'order of prokaryotes' :{'nl':u'taxon, orde van prokaryoten'},
'order of reptiles' :{'nl':u'taxon, orde van reptielen'},
'order of sea spiders' :{'nl':u'taxon, orde zeespinnen'},
'order of waterbears' :{'nl':u'taxon, orde van beerdiertjes'},
'order of worm' :{'nl':u'taxon, orde van wormen'},
'tribe of arachnids' :{'nl':u'taxon, geslachtengroep van spinachtigen'},
'tribe of insects' :{'nl':u'taxon, geslachtengroep van insecten'},
'tribe of mammals' :{'nl':u'taxon, geslachtengroep van zoogdieren'},
'tribe of plants' :{'nl':u'taxon, geslachtengroep van planten'},
'tribe of reptiles' :{'nl':u'taxon, geslachtengroep van reptielen'},
'section of plants' : {'nl':u'taxon, sectie van planten'},
'series of plants' : {'nl':u'taxon, reeks van planten'},
'species of alga' :{'nl':u'taxon, soort van algen'},
'species of annelid' :{'nl':u'taxon, soort van ringwormen'},
'species of amphibian' :{'nl':u'taxon, soort van amfibieën'},
'species of arachnid' :{'nl':u'taxon, soort van spinnen'},
'species of arachnids' :{'nl':u'taxon, soort van spinachtigen'},
'species of arthropods' :{'nl':u'taxon, soort van duizendpoten'},
'species of bird' :{'nl':u'taxon, soort van vogels'},
'species of brachiopods' :{'nl':u'taxon, soort van armpotigen'},
'species of bryozoan' :{'nl':u'taxon, soort van mosdiertjes'},
'species of chordates' :{'nl':u'taxon, soort van chordadieren'},
'species of cnidarian' :{'nl':u'taxon, soort van neteldieren'},
'species of crustacean' :{'nl':u'taxon, soort van kreefachtigen'},
'species of ctenophore' :{'nl':u'taxon, soort van ribkwallen'},
'species of echinoderm' :{'nl':u'taxon, soort van paddenstoelen'},
'species of entoprocts' :{'nl':u'taxon, soort van kelkwormen'},
'species of gastrotrichs' :{'nl':u'taxon, soort van buikharigen'},
'species of insect' :{'nl':u'taxon, soort van insecten'},
'species of fungus' :{'nl':u'taxon, soort van schimmels'},
'species of fish' :{'nl':u'taxon, soort van vissen'},
'species of mammal' :{'nl':u'taxon, soort van zoogdieren'},
'species of mollusc' :{'nl':u'taxon, soort van weekdieren'},
'species of myriapod' :{'nl':u'taxon, soort van duizendpotigen'},
'species of plant' :{'nl':u'taxon, soort van planten'},
'species of prokaryote' :{'nl':u'taxon, soort van prokaryoten'},
'species of reptile' :{'nl':u'taxon, soort van reptielen'},
'species of rotifers' :{'nl':u'taxon, soort van raderdieren'},
'species of sea spiders' :{'nl':u'taxon, soort van zeespinnen'},
'species of sponge' :{'nl':u'taxon, soort van sponsachtigen'},
'species of waterbears' :{'nl':u'taxon, soort van beerdiertjes'},
'species of virus' :{'nl':u'taxon, soort van virussen'},
'species of worm' :{'nl':u'taxon, soort van wormen'},
'spider family' :{'nl':u'taxon, familie van spinachtigen'},
'subfamily of arachnids' :{'nl':u'taxon, onderfamilie van spinachtigen'},
'subfamily of birds' :{'nl':u'taxon, onderfamilie van vogels'},
'subfamily of crustaceans' :{'nl':u'taxon, onderfamilie van kreeftachtigen'},
'subfamily of fishes' :{'nl':u'taxon, onderfamilie van vissen'},
'subfamily of insects' :{'nl':u'taxon, onderfamilie van insecten'},
'subfamily of mammals' :{'nl':u'taxon, onderfamilie van zoogdieren'},
'subfamily of molluscs' :{'nl':u'taxon, onderfamilie van weekdieren'},
'subfamily of plants' :{'nl':u'taxon, onderfamilie van planten'},
'subfamily of reptiles' :{'nl':u'taxon, onderfamilie van reptielen'},
'bird' :{'nl':u'taxon, vogel'},
'subgenus of insects' :{'nl':u'taxon, ondergeslacht van insecten'},
'subgenus of mammals' :{'nl':u'taxon, ondergeslacht van zoogdieren'},
'subgenus of plants' :{'nl':u'taxon, ondergeslacht van planten'},
'subtribe of insects' :{'nl':u'taxon, ondertribus van insecten'},
'subtribe of plants' :{'nl':u'taxon, ondertribus van planten'},
'superfamily of insects' :{'nl':u'taxon, superfamilie van insecten'},
'superfamily of molluscs' :{'nl':u'taxon, superfamilie van weekdieren'},
'superfamily of plants' :{'nl':u'taxon, superfamilie van planten'},
'variety of algae' :{'nl':u'taxon, variëteit van algachtigen'},
'variety of plants' :{'nl':u'taxon, variëteit van planten'},
'x!y~z':{'nl':''},
}
debugedo=False
debugedo=True
debug=True
default_query='claim[31:16521]' #all taxons
default_language = 'nl'
#global variables
items2do = 0
itemsdone= 0
missing_dict={}
def log_premature(itemno):
with codecs.open("taxon-description.prelog.csv","a", encoding="utf-8") as logfile:
logfile.write('%s\n' % (itemno))
logfile.close
def logme(verbose, formatstring, *parameters):
with codecs.open("taxon-description.log.csv", "a", encoding="utf-8") as logfile:
formattedstring = u'%s%s' % (formatstring, '\n')
try:
logfile.write(formattedstring % (parameters) )
except :
exctype, value = sys.exc_info()[:2]
print("1) Error writing to logfile on: [%s] [%s]" % (exctype, value))
verbose = True #now I want to see what!
logfile.close()
if verbose:
print(formatstring % (parameters))
def action_one_item(wditem):
global items2do
global itemsdone
global missing_dict
items2do -= 1
str1 = '{:>10d}'.format(itemsdone)
str2 = '{:>10}'.format(wditem.title())
str3 = '{:>10d}'.format(items2do)
sys.stdout.write("\r%s%s%s" % (str1, str3, str2)) #print how many items we still have to do ...
if ('nl' in wditem.descriptions):
orig_desc = wditem.descriptions['nl']
else:
orig_desc = ''
if ('en' in wditem.descriptions):
en_desc = wditem.descriptions['en']
if (en_desc in taxondescs):
data = {}
my_dict=taxondescs[en_desc]
for lang in my_dict:
if lang in wditem.descriptions:
if (wditem.descriptions[lang] in replacedesc[lang]):
data.update({'descriptions':{lang:my_dict[lang]}})
if debug:
print('Debug: %s' % data)
else:
try:
log_premature(wditem.title())
wditem.editEntity(data,summary=u'WD-taxon-description.py [[User:Edoderoobot/WD-taxon-description.py|source]]')
logme(False,'%s|%s|%s|%s|%s|%s',datetime.now().strftime("%Y-%b-%d/%H:%M:%S"),wditem.title(),'nl',orig_desc,my_dict[lang],'taxon-descript')
except:
pass
else:
if en_desc in missing_dict:
missing_dict[en_desc] += 1
else:
missing_dict.update({en_desc:1})
return 1
return 0
def wd_sparql_generator(query):
wikidatasite=pywikibot.Site('wikidata','wikidata')
generator=pg.WikidataSPARQLPageGenerator(query,site=wikidatasite)
for wd in generator:
wd.get(get_redirect=True)
yield wd
def wd_from_file():
repo=pywikibot.Site('wikidata','wikidata').data_repository()
csvfile=open('taxa4.csv','r')
for alllines in csvfile:
qitem=alllines[alllines.find('Q'):alllines.find(',')]
if (len(qitem)>0):
wditem=pywikibot.ItemPage(repo,qitem)
if (not(wditem.isRedirectPage())):
if wditem.exists():
wditem.get(get_redirect=True)
yield wditem
def main():
global itemsdone
itemsdone = 0
print ("main")
query = default_query #later, I want to manage this with params
lng = default_language
pigenerator = wd_sparql_generator(u'select ?item where {?item wdt:P31 wd:Q16521}')
#pigenerator = wd_from_file()
for wditem in pigenerator:
action_one_item(wditem)
itemsdone += 1
#if itemsdone > 25 : break
print('Items done: %s' % itemsdone)
if __name__ == "__main__":
if debugedo:
print("debug is on")
site=pywikibot.Site('nl')
repo=site.data_repository()
wd = pywikibot.ItemPage(repo,'Q17979303')
wd.get(get_redirect=True)
action_one_item(wd)
else:
print("Klaar voor de start")
main()
