|
#!/usr/bin/env python
|
|
|
|
"""
|
|
This is a template on how to construct a new Appion Script from scratch
|
|
|
|
At the top you have the '#!/usr/bin/env python'
|
|
|
|
this tells the computer that this is a python program and it tells it
|
|
to run the env program in the /usr/bin directory to find the location
|
|
of the python interpreter
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
import math
|
|
from appionlib import appionScript
|
|
from appionlib import apDisplay
|
|
from appionlib import apStack
|
|
from appionlib import appiondata
|
|
|
|
"""
|
|
Next, you have the import commands
|
|
|
|
You may not know which module you will need, so start with a couple
|
|
and add more as you need them. For an Appion Script you will at least
|
|
need the 'appionScript' module (or file) and the 'os' module is also
|
|
very handy. 'apDisplay' is used to print messages, wanrings and errors.
|
|
apStack will be used to get information about a stack.
|
|
|
|
For Appion libraries we use the system 'from appionlib import X'. This
|
|
tells python to go into the appionlib directory and read the file 'X.py'.
|
|
"""
|
|
|
|
"""
|
|
Now we need to create a class for our program that will inherit all the
|
|
properties of appionScript.
|
|
|
|
In this case I gave my program the name 'ExampleScript' and then put
|
|
appionScript.AppionScript in parentheses. The latter part tells python
|
|
that my new class called ExampleScript will inherit all properties from
|
|
AppionScript located in the appionScript module (or file). The
|
|
appionScript.AppionScript is required for an Appion Script.
|
|
|
|
Within a new Appion Script class there are four required functions:
|
|
setupParserOptions(), checkConflicts(), setRunDir(), and start(). Beyond
|
|
this you can create any new function that you would like and there are two
|
|
other special functions that you can override, OnInit() and OnClose(). These
|
|
function are described in more detail below.
|
|
|
|
Typically, I like to keep setupParserOptions(), checkConflicts(), setRunDir()
|
|
at the top of the class and start() as the last function in the class. This
|
|
makes it easy to find them when reading another person's code.
|
|
"""
|
|
|
|
#=====================
|
|
#=====================
|
|
class ExampleScript(appionScript.AppionScript):
|
|
#=====================
|
|
def setupParserOptions(self):
|
|
"""
|
|
This function is used to define any command line arguments.
|
|
Things like stackid, modelid, templateid, or shuffle files.
|
|
|
|
As part of Appion Script, several global variables are already defined:
|
|
runname, description, projectid, rundir, and commit. Of these only
|
|
runname is required
|
|
|
|
The commandline is parsed using the OptParse library in python,
|
|
see http://docs.python.org/library/optparse.html for documentation.
|
|
|
|
There are three major types of options: (1) strings, floats, and ints,
|
|
(2) true/false, and (3) choosing from a list.
|
|
|
|
The first type of option is obtaining a string, float, or int from the user.
|
|
It has the following format:
|
|
"""
|
|
self.parser.add_option("-s", "--old-stack-id", dest="stackid", type="int",
|
|
help="Stack database id", metavar="ID")
|
|
|
|
self.parser.add_option("--trials", dest="trials", type="int",
|
|
help="Number of 5000 particle trials", metavar="INT", default=1)
|
|
|
|
#=====================
|
|
def checkConflicts(self):
|
|
"""
|
|
After you have set the options you want to have, you need to check to make sure
|
|
they are valid and if they are required.
|
|
|
|
For this script, we'll say that the stackid is required
|
|
"""
|
|
if self.params['stackid'] is None:
|
|
apDisplay.printError("Please provide a user id, e.g. --stackid=15")
|
|
|
|
#=====================
|
|
def onInit(self):
|
|
"""
|
|
Advanced function that runs things before other things are initialized.
|
|
For example, open a log file or connect to the database.
|
|
"""
|
|
return
|
|
|
|
#=====================
|
|
def onClose(self):
|
|
"""
|
|
Advanced function that runs things after all other things are finished.
|
|
For example, close a log file.
|
|
"""
|
|
return
|
|
|
|
def commitStackData(self, params, newname=False, centered=False, oldstackparts=None, sorted=False):
|
|
oldstackdata = apStack.getOnlyStackData(params['stackid'], msg=False)
|
|
|
|
#create new stack data
|
|
stackq = appiondata.ApStackData()
|
|
stackq['path'] = appiondata.ApPathData(path=os.path.abspath(params['rundir']))
|
|
stackq['name'] = oldstackdata['name']
|
|
|
|
# use new stack name if provided
|
|
if newname:
|
|
stackq['name'] = newname
|
|
|
|
stackdata=stackq.query(results=1)
|
|
|
|
if stackdata:
|
|
apDisplay.printWarning("A stack with these parameters already exists")
|
|
return
|
|
stackq['oldstack'] = oldstackdata
|
|
stackq['hidden'] = False
|
|
stackq['substackname'] = params['runname']
|
|
stackq['description'] = params['description']
|
|
stackq['pixelsize'] = oldstackdata['pixelsize']
|
|
stackq['boxsize'] = oldstackdata['boxsize']
|
|
if 'correctbeamtilt' in params.keys():
|
|
stackq['beamtilt_corrected'] = params['correctbeamtilt']
|
|
if sorted is True:
|
|
stackq['junksorted'] = True
|
|
if centered is True:
|
|
stackq['centered'] = True
|
|
if 'mask' in params:
|
|
stackq['mask'] = params['mask']
|
|
if 'maxshift' in params:
|
|
stackq['maxshift'] = params['maxshift']
|
|
|
|
## insert now before datamanager cleans up referenced data
|
|
stackq.insert()
|
|
return stackq
|
|
|
|
def commitStackParticles(self, params, newname=False, centered=False, oldstackparts=None, sorted=False):
|
|
|
|
#Insert particles
|
|
listfile = params['keepfile']
|
|
|
|
### read list and sort
|
|
f=open(listfile,'r')
|
|
listfilelines = []
|
|
for line in f:
|
|
sline = line.strip()
|
|
if re.match("[0-9]+", sline):
|
|
listfilelines.append(int(sline.split()[0])+1)
|
|
else:
|
|
apDisplay.printWarning("Line in listfile is not int: "+str(line))
|
|
listfilelines.sort()
|
|
total = len(listfilelines)
|
|
f.close()
|
|
|
|
## index old stack particles by number
|
|
part_by_number = {}
|
|
if oldstackparts is not None:
|
|
for part in oldstackparts:
|
|
part_by_number[part['particleNumber']] = part
|
|
|
|
apDisplay.printMsg("Inserting stack particles")
|
|
count = 0
|
|
newpartnum = self.last_number + 1
|
|
t0 = time.time()
|
|
for origpartnum in listfilelines:
|
|
count += 1
|
|
if count % 100 == 0:
|
|
sys.stderr.write("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b")
|
|
sys.stderr.write(str(count)+" of "+(str(total))+" complete")
|
|
|
|
# Find corresponding particle in old stack
|
|
# Use previously queried particles if possible, otherwise
|
|
# do new query here (very slow if millions of prtls in DB)
|
|
try:
|
|
oldstackpartdata = part_by_number[origpartnum]
|
|
except KeyError:
|
|
oldstackpartdata = apStack.getStackParticle(params['stackid'], origpartnum)
|
|
|
|
# Insert particle
|
|
newstackq = appiondata.ApStackParticleData()
|
|
newstackq.update(oldstackpartdata)
|
|
newstackq['particleNumber'] = newpartnum
|
|
newstackq['stack'] = self.newstackdata
|
|
if params['commit'] is True:
|
|
newstackq.insert()
|
|
if count % 5000 == 0:
|
|
t1 = time.time()
|
|
apDisplay.printMsg('last 5000 time: %10.1f' % (t1-t0))
|
|
t0 = time.time()
|
|
newpartnum += 1
|
|
sys.stderr.write("\n")
|
|
if newpartnum == 0:
|
|
apDisplay.printError("No particles were inserted for the stack")
|
|
|
|
apDisplay.printMsg("Inserted "+str(newpartnum-1)+" stack particles into the database")
|
|
|
|
def commitStackRun(self, params):
|
|
apDisplay.printMsg("Inserting Runs in Stack")
|
|
runsinstack = apStack.getRunsInStack(params['stackid'])
|
|
for run in runsinstack:
|
|
newrunsq = appiondata.ApRunsInStackData()
|
|
newrunsq['stack'] = self.newstackdata
|
|
newrunsq['stackRun'] = run['stackRun']
|
|
if params['commit'] is True:
|
|
newrunsq.insert()
|
|
else:
|
|
apDisplay.printWarning("Not commiting to the database")
|
|
|
|
apDisplay.printMsg("finished")
|
|
return
|
|
|
|
#=====================
|
|
def start(self):
|
|
newname = self.params['runname']
|
|
numbers_per_trial = 5000
|
|
# make keepfile
|
|
self.params['first'] = 1
|
|
self.params['last'] = numbers_per_trial
|
|
stp = str(self.params['first'])
|
|
enp = str(self.params['last'])
|
|
fname = 'sub'+str(self.params['stackid'])+'_'+stp+'-'+enp+'.lst'
|
|
self.params['keepfile'] = os.path.join(self.params['rundir'],fname)
|
|
apDisplay.printMsg("Creating keep list: "+self.params['keepfile'])
|
|
f=open(self.params['keepfile'],'w')
|
|
for j in range(0,self.params['trials']):
|
|
for i in range(self.params['first'],self.params['last']+1):
|
|
f.write('%d\n' % (int(i)-1))
|
|
f.close()
|
|
|
|
self.newstackdata = self.commitStackData(self.params, newname, sorted=False)
|
|
#for trial in range(0,self.params['trials']):
|
|
for trial in range(0,1):
|
|
t0 = time.time()
|
|
self.last_number = self.params['trials'] * numbers_per_trial
|
|
self.commitStackParticles(self.params, newname, sorted=False)
|
|
t1 = time.time()
|
|
apDisplay.printMsg( 'trial %d time:%10.1f' % (trial,t1-t0))
|
|
self.commitStackRun(self.params)
|
|
|
|
|
|
#=====================
|
|
#=====================
|
|
if __name__ == '__main__':
|
|
examplescript = ExampleScript()
|
|
examplescript.start()
|
|
examplescript.close()
|
|
|
