import os
import os.path
from os.path import exists
import traceback
import socket
import subprocess
from subprocess import Popen, PIPE, check_output
import time
from datetime import datetime
### DOKU
# https://confluence.sec.dogado.net/display/WUM/floodGuard
# GIT: https://git.sec.dogado.net/mpa/floodguard
# Version 20221019-01
### default config
DEBUG=False
MAILSPERRUN=50
WAITTIMESEC=60
LOGFILE='/var/log/floodGuard.log' # default file for log
PATH=os.path.dirname(os.path.realpath(__file__)) + '/'
hostname=socket.gethostname() # needed to validate if the mail is incoming (only valid for mails mailbox@hostname) # WUM-2821
### function for logging
def Log(message):
msg = ""
try:
msg = str(datetime.now()) + ": " + message
f=open(LOGFILE, "a")
print (msg)
f.write(msg + "\n")
f.close()
except:
print ("Logfile " + LOGFILE + " not writable! continue. Message was: " + msg)
pass # if log cant be written then the software should proceed without log
def Debug(message):
if DEBUG:
Log("DEBUG: " + message)
### read configfile
if os.path.isfile('/etc/floodGuard.conf'): # no file to work with
f=open('/etc/floodGuard.conf', 'r')
if f.mode != 'r':
Log ("can not open holded.list for reading.")
f.Close()
else:
lines = f.readlines()
Log("read configfile")
for line in lines:
try:
obj = line.split()
flag = obj[0].lower()
value = obj[1]
if flag == "debug":
DEBUG = value.lower()=="true"
elif flag == "mailsperrun":
MAILSPERRUN = int(value)
elif flag == "waittimesec":
WAITTIMESEC = int(value)
elif flag == "logfile":
LOGFILE = value
elif flag == "python3bin":
continue
else:
Log("error by reading of config. Flag " + flag + " not recognised")
except:
Log("error by reading of config. Flag " + flag + " raised error")
### cleanup and minvalues
if os.path.exists(PATH + 'stop'):
os.remove(PATH + 'stop')
if WAITTIMESEC < 10:
WAITTIMESEC = 10
if MAILSPERRUN < 1:
MAILSPERRUN = 1
internalcyclecount = int(WAITTIMESEC / 2)
### MAINPROGRAM
def main():
Log("start floodGuard")
Log("getting domainlist (if it is a plesk-server)")
domainlist = Domainlist()
domainlistrefresh = 0;
maillist = Maillist()
### read old stand from a possibly crashed run
if os.path.isfile(PATH + 'holded.list'): # no file to work with
f=open(PATH + "holded.list", "r")
if f.mode != 'r':
Log ("can not open holded.list for reading.")
f.Close()
else:
lines = f.readlines()
Debug("holded.list has " + str(len(lines)) + " lines")
# analyse files
mailsfromfile = []
for line in lines:
data = line.split()
ID = data[1]
sender = data[0]
mailsfromfile.append(Mail(ID, sender, True))
# create list of all currently holded mailIDs
output=None
err=None
exit_code=None
try:
process = Popen(["mailq |awk '{print $1}' | grep '!' | tr -d '!'"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, err = process.communicate()
exit_code = process.wait()
except OSError as e:
Log("calling mailq caused an systemerror: " + e.strerror)
if exit_code != 0:
Log ("a softerror occured during mailq. error: " + str(err))
else:
output = output.decode("utf-8").splitlines()
# check which IDs from file are still in mailq hold
if output != None: # if no output is set, skip
if len(mailsfromfile) > 0 and len(output) > 0: # check only if both lists are not empty
for m in mailsfromfile:
for i in output:
if i == m.GetID(): # mail from holded.list is also in mailq holding
Debug("mail with ID " + i + " is valid (it exists)")
maillist.AddMail(m)
break # found - no need to search longer for this combination
f.close()
Log("restarting with " + str(maillist.CountMails()) + " currently holded ids from " + str(maillist.CountSenders()) + " senders.")
running=True
# data contains a list of all currently holded mails (from previews floodGuard-sessions)
while running:
domainlistrefresh += 1
if domainlistrefresh >= 10:
domainlistrefresh = 0
domainlist.refresh()
maillist = refresh_maillist(maillist,domainlist)
# at this point all mails that are holded by the software and still exists (as holded) plus all 'free' mails are in the maillist-object - sorted by senders
Log ("Currently mails in processing: " + str(maillist.CountMails()))
# check if user has less than 20 Mails in sendqueue and 0 holded --> remove user from monitoring
if maillist.CountSenders() > 0:
for s in maillist.GetSenders():
if s.CountMails(False) <= MAILSPERRUN and s.CountMails(True) == 0:
#Log ("Sender " + s.GetSendername() + " has no more mails that have to be watched. Remove sender from watchlist.")
maillist.RemoveSender(s)
if maillist.CountSenders() > 0:
for s in maillist.GetSenders():
if s.CountMails(False) > MAILSPERRUN:
while s.CountMails(False) >= MAILSPERRUN: # Hold mails until is reached
s.HoldNext() # hold next mail
else:
if s.CountMails(True) > 0:
while s.CountMails(False) < MAILSPERRUN:
if s.ReleaseNext() == False: # release mail - if there is no more mail then break
break
if s.CountMails(True) == 0:
maillist.RemoveSender(s)
else:
maillist.RemoveSender(s)
# added 20200724 - floodguard writes down which ids are not holded from him
try:
process = Popen(["mailq |awk '{print $1}' | grep '!' | tr -d '!'"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, err = process.communicate()
exit_code = process.wait()
except OSError as e:
Log("calling mailq caused an systemerror: " + e.strerror + "skipping creation of holded_external.list")
if exit_code != 0:
Log ("a softerror occured during mailq. error: " + str(err))
else:
output = output.decode("utf-8").splitlines()
try:
f=open(PATH + 'holded_external.list', "w")
if f.mode != 'w':
Log ("can not open holded_external.list for writing. continue")
else:
for i in output:
mail = maillist.GetMail(i)
if mail == None: # mailid is not in maillist
# count this
f.write(str(i) + '\n')
else:
if not mail.GetHolded(): # mail is in maillist and is not holded by the software
#and this
f.write(str(i) + '\n')
f.close()
except:
Log ("something went wrong by opening holded.list - continue")
# after
if maillist.CountSenders() > 0:
# save all sender with their mails
try:
f=open(PATH + 'holded.list', "w")
if f.mode != 'w':
Log ("can not open holded.list for writing. no backup from memory created. Please do not stop the software until the error is fixed")
else:
# write all lines to holded.list like <sender> <Mail-ID>
for s in maillist.GetSenders():
for m in s.GetAllMails():
if m.GetHolded() == True:
f.write(s.GetSendername() + ' ' + m.GetID() + '\n')
f.close()
except:
Log ("something went wrong by opening holded.list - continue")
else:
# delete holded.list
if os.path.exists(PATH + 'holded.list'):
os.remove(PATH + 'holded.list')
for s in maillist.GetSenders():
c = s.CountMails(True)
if c > 0:
Log ("Sender " + s.GetSendername() + " has " + str(c) + " mails that are hold back from floodGuard.")
# processing finished - waiting
for x in range(1, internalcyclecount):
if os.path.exists(PATH + 'stop'):
os.remove(PATH + 'stop')
Log("floodGuard stopped.")
running = False
break
else:
time.sleep(2)
# END from while loop
############################################################################################################################################################
def ReleaseMail(id):
Debug("release " + id)
return postsuper("-H", id)
def HoldMail(id):
Debug("hold " + id)
return postsuper("-h", id)
def postsuper(parameter, id):
FNULL = open(os.devnull, 'w')
process = Popen(["postsuper", parameter, id], stdout=FNULL, stderr=FNULL)
process.communicate()
exit_code = process.wait()
Debug(">> postsuper " + parameter + " " + id + " returns " + str(exit_code))
return exit_code == 0
def refresh_maillist(mlist,whitelist):
# create new list from this both lists
new = Maillist()
# get all mails from mlist that are holded
for s in mlist.GetSenders():
for m in s.GetAllMails():
if m.GetHolded():
new.AddMail(m)
# get all mails from mailq that are in processing
err=None
output=None
exit_code=None
try:
process = Popen(["mailq"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, err = process.communicate()
exit_code = process.wait()
except OSError as e:
Log("calling mailq caused an systemerror: " + e.strerror)
return []
if exit_code != 0:
Log ("a softerror occured during mailq. error: " + str(err.decode("utf-8")))
return []
Debug("mailq output " + str(len(output)) + " chars")
output = output.decode("utf-8").splitlines()
Debug("get_mailq has " + str(len(output)) + " lines to analyse")
skipping = False
currentMail = None
for rawline in output:
line = rawline.strip()
if skipping:
if line == "": # skip textblock (until a line is empty)
currentMail = None; # should there be an mail in processig remove it
skipping = False
continue
else:
continue
if line == "Mail queue is empty": # nothing to analyse - return
return new # empty maillist
if line == "": # end ob mailblock reached. If there is a mail then add it to list
if currentMail != None:
# check if it is an incomming mail ( set recv contains hostname)
internal = True
for r in currentMail.GetReceivers():
if hostname not in r and not whitelist.contains(r): # if hostname not in mailaddrss and whitelist does not know the mailaddr then its not internal
internal = False
if not internal:
Debug("adding mail " + currentMail.GetID() + " to global list. from: " + currentMail.GetSender() + " to: " + ','.join(currentMail.GetReceivers()))
new.AddMail(currentMail)
else:
Debug("mail " + currentMail.GetID() + " has only local receivers. skip it.")
currentMail = None
continue
firstchar = line[0] # first character of the given line
if firstchar == '-' or firstchar == '(': # or firstchar == ' ': # message from other server (like 'can not connect') - ignore it
continue
values = line.split()
if (len(values)) > 1: # this is the startline of a mailblock
try: # try to analyse the line
id = None
v1 = values[0] # id
if v1.endswith('!'): # waiting - also this is the start of a mail
Debug("found mail " + str(v1[:-1]) + " in mailq with status hold - skipping")
skipping = True
continue # mail is holded (either by this program or another - In both cases ignore it!)
else: # active [ ] or processing [*]
if v1.endswith('*'):
id = v1[:-1]
Debug("found mail " + id + " in mailq with status processing")
else:
id = v1
Debug("found mail " + id + " in mailq with status waiting")
sender = values[6]
currentMail = Mail(id, sender, False) # create mail from the first line of the block
except Exception as e: # something went wrong by analising the line
Log("ERROR: FLOODGUARD CRASHED while analysing the mailq startline from one mail: " + str(e))
Log("the line where it crashed was: " + line)
Log(str(traceback.format_exc()))
# dump the mailq for debugging
process = Popen(["mailq"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, err = process.communicate()
exit_code = process.wait()
output = output.decode("utf-8").splitlines()
Log("mailq output at this time:")
for s in output:
Log(s)
Log("I ignore this error and go on...")
pass
else: # there is only one item in this line - must be an receive-address
recv = values[0]
Debug("add receiver " + recv + " to mail " + currentMail.GetID())
currentMail.AddReceiver(recv)
return new
# holds a list of all local domains (excluding subdomains from $hostname - these are filtered out through another rule - no need to do more work than nessesary)
class Domainlist:
# set(str) domains
domains = set() # create empty set
def __init__(self):
self.refresh()
def refresh(self):
if exists("/sbin/plesk"): # this is a plesk server - get domainlist
try:
rawout = subprocess.Popen("/sbin/plesk bin domain --list|sed 's/^*.//g'|sort -u|grep -v $(hostname)$", shell=True, stdout=subprocess.PIPE)
lines = rawout.stdout.read().decode().splitlines()
if len(lines) > 0:
self.domains = set() # clear list for rebuild (so old entries can expire)
for line in lines:
self.domains.add(line.strip()) # add every domain to the set. to future-dev: here you can add checks if you want to filter some out
Log("refreshed domainlist. Got " + str(len(self.domains)) + " domains from plesk.")
except Exception as e: # something went wrong
Log("This is a plesk server, but something went wrong by getting the domainlist: " + str(e))
self.domains = set() # go ahead with empty set.
else: # no plesk
self.domains = set()
# can get a domain or a mailaddr and checks if the domain(or addr) is in the list
def contains(self, string):
if "@" in string: # got a mailaddr
domain = string[string.rfind("@") + 1:] # cut after last occurence of @ and get last part (domainpart)
return domain in self.domains
else: # must be a domain
return string in self.domains
class Maillist:
# set(Sender) senderlist
def __init__(self):
self.senderlist = set()
def SenderExists(self, sendername):
for s in self.sender:
if s.GetSendername() == sendername:
return True
return False
# adds a new mail to the sender. If the sender does not exists it will be created.
def AddMail(self, mail):
for s in self.senderlist:
if mail.GetSender() == s.GetSendername():
s.AddMail(mail)
return
# if he comes here the sender is new
s = Sender(mail.GetSender())
s.AddMail(mail)
self.senderlist.add(s)
return
def RemoveSender(self, sendername):
for s in self.senderlist:
if s.GetSendername() == sendername:
self.senderlist.discard(s)
return
# if sender does not exist than there is nothing to do - return
return
# sum of all mailds of all senders regardless of their status
def CountMails(self):
c=0
for s in self.senderlist:
c = c + s.CountAllMails()
return c
def CountSenders(self):
return len(self.senderlist)
def GetSenders(self):
return self.senderlist
def GetMail(self, ID):
for s in self.senderlist:
for m in s.GetAllMails():
if m.GetID() == ID:
return m
return None
class Sender:
# str sendername # will be set by init
# set(Mail) mails
def __init__(self, sendername):
self.sendername = sendername
self.mails = set()
# add a mail to this sender. If the mailid exists it will be overwritten
def AddMail(self, mail):
for m in self.mails:
if m.GetID() == mail.GetID():
# m = mail
return
self.mails.add(mail)
def Remove(self, mail):
for m in self.mails:
if m.GetID() == mail.GetID():
# self.mails.remove(m)
self.mails.discard(m)
def CountAllMails(self):
return len(self.mails)
def CountMails(self, holdstatus): # holdstatus is True or False
c=0
for m in self.mails:
if m.GetHolded() == holdstatus:
c+=1
return c
def HoldNext(self):
for m in self.mails:
if m.GetHolded() == False:
HoldMail(m.GetID())
m.SetHolded(True)
return
return
def ReleaseNext(self):
for m in self.mails:
if m.GetHolded() == True:
ReleaseMail(m.GetID())
m.SetHolded(False)
return True
return False
def GetAllMails(self):
return self.mails
def GetSendername(self):
return self.sendername
class Mail:
# str id
# str sender
## set(str) receiver
# bool holded
def __init__(self, id, sender, holded):
self.id = id
self.sender = sender
self.holded = holded
self.receiver = set()
def SetHolded(self, holded):
self.holded = holded
def GetHolded(self):
return self.holded
def GetID(self):
return self.id
def GetSender(self):
return self.sender
def AddReceiver(self, recv):
self.receiver.add(recv)
def GetReceivers(self):
return self.receiver
if __name__ == "__main__":
try:
main()
except Exception as e: # something went terribly wrong
Log("ERROR: FLOODGUARD CRASHED: " + str(e))
Log(str(traceback.format_exc()))
# dump the mailq for debugging
process = Popen(["mailq"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, err = process.communicate()
exit_code = process.wait()
output = output.decode("utf-8").splitlines()
Log("mailq output at this time:")
for s in output:
Log(s)
Log("I am dieing now...")
Mini Shell