File contents

#!/usr/bin/env python2.5
"""SpamCop Denouncer v0.2 (SCD)
Computer aided easy spam reporting.
Daniele (mythsmith) Paganelli @ 2008
Public Domain
http://daniele.modena1.it/code/spamcop-denouncer
"""
#####################
# EDIT THESE SETTINGS
#####################
# SpamCop settings
user_name='XXXX'	# spamcop.net login information
password='XXXX'

max_age=12	# Accept only messages more recent than max_age (in hours). Spamcop.net internal maximum is 12h. 
notes=''	# Standard optional notes to send to every recipient of your report (like a signature). You have opportunity to change it for every message

# ALARM: alert when one of these domain is found as spam source (spamvertised) or in report recipients
alarm=['modena1','agescimo','homelinux.org','seeweb']


# Special paths
status_file='/home/daniele/.spamcop.dat'	# Where SCD stores the cookie and id cache.
tmp='/tmp/denounce'			# Temporary file for downloaded mail
tmpdir='/tmp'				# Temp dir where to store cached repors
read_file='/var/mail/cop'		# Default file to scan for spamcop urls
browser='less '				# Command to view the detailed spam report in textual form
box='/home/daniele/.kde/share/apps/kmail/mail/Spambox'

# Tweaks
width=100	# Dialog width (adapt to your terminal)
idcache=500	# Max number of IDs and hashes to remember in the status_file
delay=7		# delay in paste mode for non-paying reporters

# Optional email settings for fetchmail (-f)
protocol='IMAP'		# 'IMAP' or 'POP3'
ssl=True
IMAP_user=''
IMAP_server=''
IMAP_folder=''
POP3_user=''
POP3_server=''


# END OF SETTINGS
#------------------

# Default switches (warning: these will modify the behaviour of command line switches!)
keep=True
safe=True
fetch=False
read=False
clearcache=False
debug=False
idmode=False
paste=False
mbox=False
unreported=False
queue=False

##############
# PROGRAM
##############

from httplib import HTTPConnection,HTTP
from sys import argv,exit
from urllib import urlencode,unquote_plus
from commands import getstatusoutput as go
from os import popen,remove,system,environ
from subprocess import Popen,PIPE
from sys import exit,stdin,stdout,stderr
from optparse import OptionParser
from cStringIO import StringIO
from formatter import AbstractFormatter,DumbWriter
from htmllib import HTMLParser
from time import sleep,strftime,time
import hashlib

version='0.2'
help="""
SpamCop Denouncer v%s
http://daniele.modena1.it/code/spamcop-denouncer
----------------------

 -h --help			Print this help
 -v 				verbose

SPAM SOURCE OPTIONS:
 -i <id,id...> --id=<id,id...>	Process comma-separed SpamCop IDs
 -q	--queue			Process all your unreported spam at spamcop.net queue (very slow)
 -f --fetch			Download new email to a file and get SpamCop IDs from it.
 -r <path> --read=<path>	Read from file <path> instead of downloading/pasting
 -p --paste			Read file is standard input
 -m <mbox> --mbox <mbox>	Read an mbox of spam messages and report them via the spamcop web interface.


REPORTING OPTIONS:
 -n <msg> --notes=<msg>		Send <msg> within each report
 -s --safe			Safe reporting (default)
 -u --unsafe			VERY DANGEROUS - Automated reporting

EXIT OPTIONS:
 --no-keep --nk -d		Don't keep mail/read file/mbox
 -k --keep				Keep downloaded mail/read file/mbox after reporting (default)
 -c --clear			Clear the SpamCop ID and hash cache (forget about already processed IDs)

STATISTICS:
	-S<d[,t]> --stats=<d[,t]>	Print statistics collected since the first run date, then exit.
It will print some global stats, the reporting activity by <d>-days periods, the top <t=10> report recipent's domains.
The most significant data are the SHI cost estimations. They mean "Seconds of Human Interaction", calculated
from the _response_ to the first spam analysis prompt (dialog) till the end of the analysis. 
SHI values resume the time wasted due to the reporting activity.

MUTUALLY EXCLUSIVE OPTIONS:
These options cannot be set at the same time:
	fetch, read, idmode / f, r, i
	fetch, mbox, idmode / f, m, i
	fetch, read, paste / f, r, p
	keep, id / k, i
	keep, paste / k, p

EXAMPLES:
Using: --paste --mbox. , you can paste your spam message to stdin and report it as you would do with the web form.

NOTICE: 
If any <var>=. the defaults will be used. Example: spamcop -r. , will use the read file option 
hardcoded in the program header.
""" %version




environ["DIALOGOPTS"]='--single-quoted --colors --backtitle "SpamCop Denouncer v%s ' % version
ua='User-Agent',"SpamCop Denouncer/%s (+http://daniele.modena1.it/code/spamcop-denouncer)" % version

# If this string is found in a spamcop page, it means the cookie-login has expired:
relogin=('<input size=15 type="text" name="username">\n<input size=8 type="password" name="password">\n<input type="submit" value="Login">',
'<div class="error">No userid found, sorry.</div>')

submit_form={'action':'submit','oldverbose':'0','spam':None,'x1':'Process Spam','verbose':'1'}

htime=None

hash_cache=idcache	# Max number of spam hashes to remember

# add this string if a message is empty (but has headers)
bodyless='This spam email did not have a body. This string was added for reporting purposes' 

### Utilities
def f(s,si,se):
	"""Get a slice of s which begins with si and ends with se"""
	i=s.find(si)+len(si)
	if i<0:
		return False
	e=s.find(se,i)
	if e<0:
		return False
	return s[i:e]


def multipart(fields):
    """From http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306"""
    BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
    CRLF = '\r\n'
    L = []
    for key in fields.keys():
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"' % key)
	L.append('Content-Type: text/plain')
        L.append('')
        L.append(fields[key])
    L.append('--' + BOUNDARY + '--')
    L.append('')
    body = CRLF.join(L)
    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
    return content_type, body

def htdecode(a):
	"""Simple html 2 text. Thanks to Michel Claveau: http://www.thescripts.com/forum/post86944-4.html"""
	f=StringIO()
	z=AbstractFormatter(DumbWriter (f))
	p=HTMLParser(z)
	p.feed(unquote_plus(a))
	p.close()
	sret=f.getvalue()
	f.close()
	return(sret)

# Thanks to Daniel Yoo, http://mail.python.org/pipermail/python-list/2004-September/284991.html
def backfileiter(myfile):
    """Iterates the lines of a file in reverse order."""
    myfile.seek(0)
    offsets = _getLineOffsets(myfile)
    myfile.seek(0)
    offsets.reverse()
    for i in offsets:
        myfile.seek(i+1)
        yield myfile.readline()

def _getLineOffsets(myfile):
    """Return a list of offsets where newlines are located."""
    offsets = [-1]
    i = 0
    while True:
        byte = myfile.read(1)
        if not byte:
            break
        elif byte == '\n':
            offsets.append(i)
        i += 1
    return offsets
###

def post(act,params,cl=True):
	"""Cookie aware POST"""
	#body=urlencode(params)
	content_type,body=multipart(params)
	conn=HTTP('www.spamcop.net')
	conn.putrequest('POST', act)
	conn.putheader('Host', 'www.spamcop.net')
	#conn.putheader("Content-type", "application/x-www-form-urlencoded")
	conn.putheader("Content-type", content_type)
	if cl:
		conn.putheader('Cookie', cookie )
	conn.putheader("Content-length", "%d" % len(body))
	conn.putheader(ua[0],ua[1])
	conn.endheaders()
	conn.send(body)
	reply, msg, hdrs = conn.getreply()
	page = conn.getfile().read()
	for logerr in relogin:
		if logerr in page:
			print 'Cookie expired.'
			login()
			post(act,params,cl)
	conn.close()
	return reply,msg,hdrs,page

def login():
	"""Get a new cookielogin"""
	global cookie
	print 'Requesting a new authentication...'
	params={'username':user_name,'password':password,'duration':'+1y','action':'cookielogin','returnurl':'/'}
	reply,msg,hdrs,page=post('/mcgi',params,cl=False)
	cookie=hdrs['Set-Cookie'].split(';')[0]
	return reply,msg,hdrs,page
	
def get(act):
	"""Cookie aware GET"""
	conn=HTTP('www.spamcop.net')
	conn.putrequest('GET', act)
	conn.putheader('Host', 'www.spamcop.net')
	conn.putheader('Cookie', cookie )
	conn.putheader(ua[0],ua[1])
	conn.endheaders()
	reply, msg, hdrs = conn.getreply()
	page = conn.getfile().read()
	for logerr in relogin:
		if logerr in page:
			print 'Cookie expired.'
			login()
			get(act)
	conn.close()
	return reply,msg,hdrs,page

meantime=0
def parameters(page):
	"""Parse the reporting page to find out which are the reporting options"""
	global unreported,meantime
	cancel=False
	if 'Please wait - subscribe to remove this delay' in page:
		return (2,{},'delayed')
	if  'Reports regarding this spam have already been sent:' in page:
		return (1,{},'report already sent')
	if '<div class="error">No body text provided, check format of submission.' in page:
		return (1,{},'body text not found, check format of submission')
	if '<div class="error">ISP has indicated spam will cease; ISP resolved this issue' in page:
		return (1,{},'ISP has indicated spam will cease. Report cancelled.')
	old=f(page,'Message is','hours old')
	try: old=int(old.replace(' ',''))
	except:
		print '_/!\\_ Error parsing spam age. Continuing...'
		old=-1
	if old>max_age:
		cancel=True
	olds=''
	if old>=0 and old<=max_age:
		olds='Message is %i hours old' %old
	if 'Sorry, this email is too old to file a spam report.' in page:
		return (1,{},'this email is too old to file a spam report. '+olds)
	if 'Yum, this spam is fresh!' in page:
		print '\tYum, this spam is fresh! '+olds
	else: print '\t'+olds
	if old<max_age and old>=0:
		meantime+=old
	if not unreported:
		if "Unreported Spam Saved" in page:
			unreported=True
	params={}
	assure=f(page,'Please make sure this email IS spam:','</font><a')
	if assure:
		params['ASSURE']='\n'+htdecode(assure).replace('"',"\\\"").replace('`',"'").replace('$','\$')
	else: params['ASSURE']=''
	page=f(page,'<form action="/sc"','</form>')
	if page==False:
		return (1,{},'input form not found! Please check the ID manually.')
	page=page.split('<input ')[1:]
	
	for line in page:
		if "type=" not in line or "name=" not in line:
			continue
		l=line.split('>')[0]
		vname=f(l,'name="','"')
		if 'value=' in l:
			vval=f(l,'value="','"')
		elif 'checked' in l:
			vval='True'
		else:
			continue
		params[vname]=vval
	if 'reports' not in params.keys():
		return(1,params,'missing data, \n'+repr(params))
	elif type(params['reports'])==type(True):
		return(1,params,'erroneous "reports" parameter:, \n'+repr(params))
	if cancel:
		params['cancel']='Cancel'
		return (3,params,'message is older (%ih) than user defined max_age (%ih)' %(old,max_age))

	return (0,params,'OK')


def warn(rid,params,pc):
	"""Select the recipients of the report, to inspect it and add/modify a custom note."""
	global htime
	reports=params['reports'].replace(':source:',' s:').replace(':notify:',' n:').replace(':www:',' w:').split('|')
	lh=len(reports)+5
	h=lh+8+len(params['ASSURE'].splitlines())
	dialog='dialog --extra-button --extra-label "Back" --title "[%.1f%%] Analysing id=%s" --checklist \
	"  Please \Z4\Zuassure this email IS spam:\Zn %s" %i %i %i v "\ZuView Full Report" off w "\ZbAdd/modify optional notes" off ' % (pc,rid,params['ASSURE'],h,width,lh)
	msgdialog='dialog --title "Add/modify optional notes" --inputbox "Enter your notes for the recipients" 10 %i "%s"'
	rn=1
	for r in reports:
		s='on'
		if params['send%i' % rn]!='True':
			s='off'
		for al in alarm:
			if al in r:
				print '\a'
		dialog+='%i "%s" %s ' %(rn,r,s)
		rn+=1
	# Ask what to do
	sel='v'
	msg=notes
	dmsg=''
	if msg!='':
		dmsg=' m "%s" on ' % msg
	while 'v' in sel or 'w' in sel:
		print dialog+dmsg
		diapro=Popen(dialog+dmsg,shell=True,stderr=PIPE)
		sel=diapro.stderr.read()
		exitcode=diapro.wait()
		print 'exitcode: ',exitcode
		if exitcode==3:
			return 'PREVIOUS_SPAM'
		if htime==None: 	# If this is the first human interaction
			htime=time()	# the htime timer must be initialized
		print sel
		if 'v' in sel:
			tmpage=tmpdir+'/spamcop-'+rid+'.txt'
			Popen(browser+tmpage,shell=True,stderr=PIPE).stderr.read()
		if 'w' in sel:
			msg=Popen(msgdialog % (width,msg),shell=True,stderr=PIPE).stderr.read().replace('"','\\"')
			print msg
			if msg!='':
				dmsg=' m "%s" on ' % msg
			else: dmsg=''
	params['cancel']=False
	if sel=='':
		params['cancel']='Cancel'
		return params
	#Remove unwanted parameters:
	sel=sel.split(' ')
	for p in params.keys():
		if p[:4]=='send' and p[4:] not in sel:
			pn=p[4:]
			params[p]=params['type'+pn]=\
			 params['master'+pn]=params['info'+pn]=\
			 params['sc_comment'+pn]=None
			del params[p],params['type'+pn],\
			 params['master'+pn],params['info'+pn],\
			 params['sc_comment'+pn]
	if msg!='' and 'm' in sel:
		params['notes']=msg
	return params

def response(page):
	"""Read which reports were sent"""
	i=page.find('<div id="content">')+len('<div id="content">')
	e=page.find('<p>',i)
	page=page[i:e].replace('<br>','').replace('<div class="error">','_/!\\_ Error: ').replace('</div>','')
	if 'Report canceled.' in page:
		page='\t Report cancelled.'
	return page

def fetchmail():
	print 'Fetching new mail to %s' %read_file
	exec('u=%s_user' %protocol)
	exec('srv=%s_server' %protocol)
	plus=''
	if protocol=='IMAP' and IMAP_folder:
		plus='--folder '+IMAP_folder
	if ssl:
		plus+=' --ssl '
	cmd="fetchmail -Uk -p %s --username %s %s --bsmtp %s %s" % (protocol,u,plus,read_file,srv)
	system(cmd)

def grep_urls(f):
	f=open(f,'r')
	urls=[]
	skip=0
	lc=0
	for line in backfileiter(f):
		lc+=1
		if line[:29]=="http://www.spamcop.net/sc?id=":
			rid=line.split('=')[1].replace('\n','')
			if rid not in cache and rid not in urls:
				urls+=[rid]
			else: 
				skip+=1
			if skip>=idcache:
				print '\nIDCache limit of %i reached at line -%i' %(idcache,lc)
				break
	if skip>0:
		print '\nIgnored %i SpamCop IDs already in cache.' %skip
	f.close()
	return urls

submitted=0

def submit(email):
	"""Submits <email> to spamcop webform and returns rid, page, params"""
	global submitted
	submit_form['spam']=email
	reply,msg,hdrs,page=post('/sc',submit_form)
	if reply!=302:
		print '_/!\\_ Submit error: ',reply,msg
		return False,'Connection Error'
	submit_form['spam']=None
	rid=hdrs['location'].split('=')[1]
	shash=hashlib.md5(email).hexdigest()
	if shash not in hache:
		submitted+=1
		print '%i Submit hash: %s Got ID: %s' %(submitted,shash,rid)
		return rid,shash
	else:
		
		print '_/!\\_ Already submitted: the message is in hash cache with ID %s' % cache[hache.index(shash)]
		return False,'Hash in cache'

def queue_next():
	reply,msg,hdrs,page=get('/')
	if msg!='OK':
		print '_/!\\_Connection error: %s, %s' %(reply,msg)
		return 1
	if "Unreported Spam Saved" not in page:
		print 'End queue'
		return False
	rid=f(page,'<a href="/sc?id=','">')

	return rid

cache=[]
hache=[]
stats=[]
cookie='code=a'
def database():
	"""Read cookie, id cache, hash cache and statistics"""
	global cookie,cache,hache,stats
	try:
		status=open(status_file,'r').read().splitlines()
		cookie=status[0]
		h=0
		if len(status)>3 and not clearcache:
			for l in status[2:]:
				if l in ['---HASH---']:
					h=1
					continue
				if l in ['---STATS---']:
					h=2
					continue
				if h==0: cache+=[l]
				elif h==1: hache+=[l]
				elif h==2: stats+=[l]
	except:
		cookie='code=a'

def print_stats(each,top):
	"""Output some statistics"""
	collect={}; dstats={}
	meanP=0; tP=0
	meanS=0; tS=0
	meanT=0; 
	hcost=0.; mcost=0.
	for s in stats: # Aggregate stats
		sv=s.replace('\t','').replace(' ','').split(';')
		day=sv[1].split('/')
		day=int(day[0])*30+int(day[1])+int(day[2])*365
		day=day//each
		if day not in collect.keys():
			collect[day]=[sv[1],0,0,0,0.,0]
		collect[day][1]+=int(sv[2])
		tP+=int(sv[2])
		collect[day][2]+=int(sv[4])
		tS+=int(sv[4])
		collect[day][3]+=1
		collect[day][4]+=float(sv[6])
		hcost+=float(sv[6])
		meantime=int(sv[7])
		meanT+=meantime
		collect[day][5]+=meantime
		for ndom in sv[8].split(':'): # Aggregate domain stats
			if '@' not in ndom: continue
			[n,dom]=ndom.split('@')
			if dom not in dstats.keys():
				dstats[dom]=0
			dstats[dom]+=int(n)
	mcost=hcost/tS
	meanT=1.*meanT/tS
	meanP=1.*tP/len(collect.keys())
	meanS=1.*tS/len(collect.keys())
	sday=collect.keys()
	sday.sort()
	pe='days'
	if each==1: pe='day'
	print '\nSTATISTICS since %s (%i %s period):' % (collect[sday[0]][0],each,pe)
	print 'Reporting quality: %7.3fh of mean spam age' % meanT
	print 'Total time cost: %6.1fshi, efficiency: %6.1fs/spam'% (hcost,mcost)
	print 'Processed: %i, %5.1f/period - Reported: %i, %5.1f/period' % (tP,meanP,tS,meanS)
	
	print '\nReporting activity (%i %s period):' % (each,pe)
	print '%-8s   %-9s  %-8s  %-8s  %-6s  %-7s' % ('day','processed','reported','sessions','cost','quality')
	for day in sday:
		d=collect[day]
		d[-1]=1.*d[-1]/d[2]
		print '%-8s   %-9i  %-8i  %-8i  %6.2f  %6.2f' % tuple(d)
	
	print '\nOverall top %i report destinations' % top
	dk=dstats.keys()
	dk.sort(lambda x,y: dstats[y]-dstats[x])
	rep=len(dk)
	if rep>top:
		rep=top
	for i in range(rep):
		print '%-25s %i' % (dk[i],dstats[dk[i]])
	print

#####################################################
# Process

Reports=[]
OkReports=[]
c=0
def process_urls(urls):
	global Reports,c,OkReports
	tot=len(urls)
	print '\nProcessing %i spamcop report links:' % tot	
	for rid in urls:
		c+=1
		# url from input
		url='/sc?id='+rid
		print '\n[%.1f%%] Processing SpamCop ID: %s' % (100.*c/tot,rid)
		# get reporting page
		status=2
		while status==2:
			reply,msg,hdrs,page=get(url)
			if msg!='OK':
				print '_/!\\_ Connection Error: %s %s' % (reply,msg)
				continue
			# extract parameters
			status,params,sm=parameters(page)
			if status==2:
				print 'Delayed %i seconds'   %delay
				sleep(delay)
			
		if status==1:	
			print '_/!\\_ Parsing error: ',sm
			continue
		elif status==3:
			print '_/!\\_ Removing: ',sm
			OkReports+=[(rid,params)]
			continue
		tmpage='%s/spamcop-%s.txt' % (tmpdir,rid)
		page=htdecode(page)
		open(tmpage,'w').write(page)
		Reports+=[(rid,params)]

#####################################################
# Analysis

cr=0
def analysis(Reports):
	global OkReports,tot,cr,htime
	rdict={}
	tot=len(Reports)
	print '\nAnalyzing %i spam-reports:' % tot
	i=0
	while i<tot:
		if i<0: i=0
		r=Reports[i]
		rid,params=r
		print '\n[%.1f%%] Analyzing SpamCop ID: %s' % (100.*i/tot,rid)
		# warn
		print params
		params=warn(rid,params,100.*i/tot)
		if params=='PREVIOUS_SPAM':
			i+=-1
			continue
		i+=1
		if params==False:
			print 'Report rejected.'
			continue
		rdict[i]=(rid,params)
	for r in Reports:
		rid,params=r
	cr+=i
	OkReports+=rdict.values()
	if htime!=None:
		htime=time()-htime



#####################################################
# Commit

cs=0
canc=0
def commit(OkReports):
	global cs,canc
	tot=len(OkReports)
	print '\nSending %i spam reports...' % tot
	for r in OkReports:
		rid,params=r
		# Remove unwanted/internal parameters:
		del params['ASSURE'], params['preview']
		if not params['cancel']:
			del params['cancel']
		print '\n[%.1f%%] Sending SpamCop ID: %s' % (100.*cs/tot,rid)
		# send reports
		url='/sc?id='+rid
		reply,msg,hdrs,page=post(url,params)
		# extract reply
		if debug:
			print params
			print page
		else:
			p=response(page)
			domain_stats(p)
			print p
		if 'cancel' in params.keys():
			if params['cancel']=='Cancel':
				canc+=1
				continue
		cs+=1
		tmpage='%s/spamcop-%s.txt' % (tmpdir,rid)
		remove(tmpage)

####################################################
# Domain stats collector

dstats={}
def domain_stats(pagesec):
	global dstats
	for t in pagesec.replace('\n',' ').split(' '):
		if '@' in t:
			dom=t.split('@')[1].replace(' ','')
			if dom not in dstats.keys():
				dstats[dom]=0
			dstats[dom]+=1
				

#############################################
# 		MAIN PROGRAM		    #
#############################################

# Option parsing
urls=[]
parser = OptionParser(conflict_handler="resolve")
parser.add_option('-S','--stats',dest='ostats',default=False)
parser.add_option('-f','--fetch',action='store_true',dest='fetch',default=fetch)
parser.add_option('-r','--read',dest='read',default=False)
parser.add_option('-s','--safe',action='store_true',dest='safe',default=safe)
parser.add_option('-u','--unsafe',action='store_false',dest='safe',default=safe)
parser.add_option('-k','--keep',action='store_true',dest='keep',default=keep)
parser.add_option('-d','--delete','--no-keep','--nk',action='store_false',dest='keep',default=keep)
parser.add_option('-c','--clear',action='store_true',dest='clearcache',default=clearcache)
parser.add_option('-n','--notes',dest='notes',default=notes)
parser.add_option('-v','--verbose',action='store_true',dest='debug',default=debug)
parser.add_option('-i','--id',dest='id',default=False)
parser.add_option('-p','--paste',action='store_true',dest='paste',default=paste)
parser.add_option('-m','--mbox',dest='mbox',default=False)
parser.add_option('-q','--queue',action='store_true',dest='queue',default=queue)
parser.add_option('-h','--help',action='store_true',dest='printhelp',default=False)

(o,a) = parser.parse_args()
if o.printhelp:
	print help
	exit(0)
fetch=o.fetch
if o.ostats:
	database()
	period=1
	top=10
	if o.ostats!='.':
		o=o.ostats.split(',')
		period=int(o[0])
		if len(o)==2: top=int(o[1])
	print_stats(period,top)
	exit(0)
if o.read:
	if o.read!='.': read_file=o.read
	read=True
safe=o.safe
keep=o.keep
clearcache=o.clearcache
notes=o.notes
debug=o.debug
if o.id:
	idmode=True
	urls=id.replace(' ','').split(',')
paste=o.paste
if o.mbox:
	mbox=True
	if o.mbox!='.': box=o.mbox
queue=o.queue

if len(argv)==1:
	ip=argv[0]


if fetch*(read + idmode +paste) + \
		idmode*(read +paste) +\
		queue*(read+idmode+fetch+paste)  != 0:
	print '_/!\\_ Error: mutually exclusive options specified.'
	print help
	exit(2)


# Read database
database()

# Get urls
if read==False:
	read_file=tmp
if fetch:
	fetchmail()

if read or fetch: 
	try: 
		urls=grep_urls(read_file)
	except IOError: 
		print '_/!\\_ Data not found.'
		exit(1)

elif paste: 
	print 'Paste a single entire spam (headers, blank line, body). Then press Ctrl+D to submit.'
	email=stdin.read()
	rid,shash=submit(email)
	if rid:
		urls=[rid]
		hache+=[shash]

# Process data
if read or fetch or paste or idmode:
	process_urls(urls)
elif queue:
	while True:
		try:
			Reports=[]
			OkReports=[]
			rid=queue_next()
			if not rid:
				'_/!\\_ Error getting the next queue object.'
				break
			elif rid==1:
				sleep(delay)
				continue
			process_urls([rid])
			if safe:
				analysis(Reports)
			commit(OkReports)
			cache+=[rid]
		except KeyboardInterrupt:
				pass
elif mbox: 
	from mailbox import mbox as Mbox
	MB=Mbox(box)
	print 'Submitting %i mail messages from mailbox %s'% (len(MB.keys()),box)
	for msg in MB:
		if len(msg.get_payload())==0 and len(msg.keys())>1:
			msg.set_payload(bodyless)
		rid,shash=submit(msg.as_string())
		if rid:
			urls+=[rid]
			hache+=[shash]
	process_urls(urls)

if safe:
	analysis(Reports)
else: 
	cr=len(Reports)
	OkReports=Reports

if not queue:
	commit(OkReports)

# Update caches
cache=urls+cache
l=len(cache)
if l>idcache:
	cache=cache[  : (l-idcache)]
l=len(hache)
if l>hash_cache:
	hache=hache[(l-hash_cache):]
sdstat=''
for d in dstats.keys():
	sdstat+='%i@%s:' % (dstats[d],d)
if c!=0:
	stats+=['%s\t; %i\t; %i\t; %i\t; %i\t; %6.2f\t; %i\t; %s'  % (strftime('%X ; %x'),c,cr,cs,canc,htime,meantime,sdstat[:-1])]

# Commit database
dat=open(status_file,'w')
dat.write(cookie+'\n---ID---\n')
for rid in cache:
	dat.write(rid+'\n')
dat.write('---HASH---\n')
for shash in hache:
	dat.write(shash+'\n')
dat.write('---STATS---\n')
for s in stats:
	dat.write(s+'\n')
dat.close()

# Print session stats
if htime==None: htime=0.
print """Done: 
	%i emails processed
	%i emails analyzed
	%i reports sent
	%i reports cancelled
Time cost: %6.2fshi""" % (c,cr,cs,canc,htime)

# Cleanup
if not keep+idmode+paste+queue+keep+mbox:
	remove(read_file)
	print 'File: %s, removed' %read_file
if mbox and not keep:
	open(box,'w').write('')
	print 'Mailbox: %s, cleared' %box