We Keep Coding

Python append text between lines

The task:
I have list of IPs which needs to be added to the .htaccess files in this format:
##ip_access1
Require ip 127.0.0.1
Require ip 127.0.0.2
Require all denied
##ip_access2
The problem:
How to append text into .htaccess file with Python? I know how to do this with bash, but I need Python specifically for now.
Cases:
If tuple of IPs is empty, find pattern ##ip_access1 ##ip_access2 and delete everything between them including the pattern in the file;
If .htaccess file is not empty, append ##ip_access1 <...> ##ip_access2 to the bottom of the file with all IPs;
P.S. Bash implementation.
ip_access() {
local user=$1
local htaccess="/var/www/${user}/site/.htaccess"
local ips="$2"
# manipulate records in .htaccess
[ ! -f "${htaccess}" ] && touch "${htaccess}"
if [ -z "${ips}" ]; then
sed -i '/##ip_access1/,/##ip_access2/{d}' "${htaccess}"
chown "${user}":"${user}" "${htaccess}"
echo "IP access successfully reset!"
exit 0
fi
arrip=()
for ip in ${ips//,/ }; do
arrip+=("Require ip $ip\n")
done
# always inject fresh batch of ips
sed -i '/##ip_access1/,/##ip_access2/{d}' "${htaccess}"
{ echo -e "##ip_access1";\
echo -e "${arrip:?}" | head -c -1;\
echo -e "Require all denied";\
echo -e "##ip_access2"; } >> "${htaccess}"
chown "${user}":"${user}" "${htaccess}"
echo "IP access successfully set!"
}

This function is the bare bones of a possible solution. It doesn't perform any sanity checks so caution should be exercised.
import os
def ips_to_file(ips, file_path):
if len(ips) > 0:
ip_lines = ['##ip_access1'] + [f'Require ip {ip}' for ip in ips] + ['Require all denied', '##ip_access2']
else:
ip_lines = []
if os.path.isfile(file_path):
with open(file_path, 'r+') as fp:
lines = [line.strip() for line in fp.readlines()]
lines = lines[:lines.index('##ip_access1')] + ip_lines + lines[lines.index('##ip_access2')+1:]
fp.seek(0)
fp.truncate()
fp.writelines(lines)

Found solution with help of course:
from typing import *
ip_lst = ["1.1.1.1", "2.2.2.2", "3.3.3.3"]
htaccess_file_contents = open("test.txt", "r").read()
def _generate_htaccess_compat_lst(lst) -> str:
to_return = []
for addr in lst:
to_return.append("Require ip " + addr)
return "\n{}\n".format("\n".join(to_return))
def _inject_between(start, end, to_manipulate, to_replace) -> str:
lines = to_manipulate.splitlines()
counter = 0
pos1, pos2 = -1, -1
# find lines between that we need to replace
for line in lines:
if start == line:
pos1 = counter
elif end == line:
pos2 = counter
counter += 1
# return null if we can't find text between
if pos1 == -1 or pos2 == -1:
return None
# +1 to offset the last line as the first index is inclusive
return "\n".join(lines[0:pos1]) + start + to_replace + end + "\n".join(lines[pos2 + 1:len(lines)])
tmp = _inject_between("##ip_access1", "##ip_access2",
htaccess_file_contents,
_generate_htaccess_compat_lst(ip_lst))
print(tmp)
# feel free to write tmp back to .htaccess

Comparing two files and removing all whitespaces

Is there a more elegant way of comparing these two files?
Right now I am getting the following error message: syntax error near unexpected token (... diff <( tr -d ' '.
result = Popen("diff <( tr -d ' \n' <" + file1 + ") <( tr -d ' \n' <"
+ file2 + ") | wc =l", shell=True, stdout=PIPE).stdout.read()
Python seems to read "\n" as a literal character.

The constructs you are using are interpreted by bash and do not form a standalone statement that you can pass to system() or exec().
<( ${CMD} )
< ${FILE}
${CMD1} | ${CMD2}
As such, you will need to wire-up the redirection and pipelines yourself, or call on bash to interpret the line for you (as #wizzwizz4 suggests).
A better solution would be to use something like difflib that will perform this internally to your process rather than calling on system() / fork() / exec().
Using difflib.unified_diff will give you a similar result:
import difflib
def read_file_no_blanks(filename):
with open(filename, 'r') as f:
lines = f.readlines()
for line in lines:
if line == '\n':
continue
yield line
def count_differences(diff_lines):
diff_count = 0
for line in diff_lines:
if line[0] not in [ '-', '+' ]:
continue
if line[0:3] in [ '---', '+++' ]:
continue
diff_count += 1
return diff_count
a_lines = list(read_file_no_blanks('a'))
b_lines = list(read_file_no_blanks('b'))
diff_lines = difflib.unified_diff(a_lines, b_lines)
diff_count = count_differences(diff_lines)
print('differences: %d' % ( diff_count ))

This will fail when you fix the syntax error because you are attempting to use bash syntax in what is implemented as a C system call.
If you wish to do this in this way, either write a shell script or use the following:
result = Popen(['bash', '-c',
"diff <( tr -d ' \n' <" + file1 + ") <( tr -d ' \n' <"
+ file2 + ") | wc =l"], shell=True, stdout=PIPE).stdout.read()
This is not an elegant solution, however, since it is relying on the GNU coreutils and bash. A more elegant solution would be pure Python. You could do this with the difflib module and the re module.

'The filename, directory name, or volume label syntax is incorrect.\r\n', None

I am trying to have a Python script stage a file to a locally hosted Git:
gitExe = r"C:\Git\bin\git.exe"
gitdir = r' --git-dir="G:\QA\.git"'
worktree = r' --work-tree="G:\QA\"'
gitcmd = " add"
myCsv = "Reports.csv"
import subprocess
print gitdir
print gitcmd
print gitExe
cmd = 'C:\\Git\\bin\\git.exe --git-dir="G:\\QA\\.git" --work-tree="G:\\QA" add Reports.csv' ####WORKS####
#cmd = ['C:\\Git\\bin\\git.exe --git-dir="G:\\QA\\.git" --work-tree="G:\\QA" add Reports.csv'] ####DOESNT WORK####
#cmd = ['C:\\Git\\bin\\git.exe --git-dir="G:\\QA\\.git" --work-tree="G:\\QA" add Reports.csv'+""] ####DOESNT WORK####
#cmd = [gitExe + gitdir + worktree + gitcmd + myCsv] ####DOESNT WORK####
p = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
print cmd
print p.communicate()
When I pass the full command as a string (uncommendted line above) everything works fine but when I pass it as an array (the three commented out lines above) I get the following error message:
>>> ================================ RESTART ================================
>>>
--git-dir="G:\QA\.git"
add
C:\Git\bin\git.exe
['C:\\Git\\bin\\git.exe --git-dir="G:\\QA\\.git" --work-tree="G:\\QA\\" addReports.csv']
('The filename, directory name, or volume label syntax is incorrect.\r\n', None)
What gives?

When you pass the command as an array, each parameter must be a separate element.
cmd = ['C:\\Git\\bin\\git.exe', '--git-dir="G:\\QA\\.git"', '--work-tree="G:\\QA"',
'add', 'Reports.csv']
The first item in the array is the program name. In your case, python tried to make the entire command line into the program name.

Python using zenity with var

I'm trying to run Zenity in a python script, to display a variable.
nmaj = 10
cmd = ["zenity" "--question" "--text='Are you " + str(nmaj) + "years old ?'"]
subprocess.call(cmd, shell=True)
Can I put a string in the command? How?
Thanks

You can try using format and putting '' outer than "":
nmaj = 10
cmd = 'zenity --question --text="Are you {} years old ?"'.format(nmaj)
subprocess.call(cmd, shell=True)

You miss comma(,)s between command line arguments:
cmd = ["zenity", "--question", "--text='Are you " + str(nmaj) + "years old ?'"]
Otherwise the string literals are concatenated into a string (See String literal concatenation):
>>> "zenity" "--question" "--text='Are you "
"zenity--question--text='Are you "

Python script that performs line matching over stale files generates inconsistent output

I created a python script to parse mail (exim) logfiles and execute pattern matching in order to get a top 100 list for most send domains on my smtp servers.
However, everytime I execute the script I get a different count.
These are stale logfiles, and I cannot find a functional flaw in my code.
Example output:
1:
70353 gmail.com
68337 hotmail.com
53657 yahoo.com
2:
70020 gmail.com
67741 hotmail.com
54397 yahoo.com
3:
70191 gmail.com
67917 hotmail.com
54438 yahoo.com
Code:
#!/usr/bin/env python
import os
import datetime
import re
from collections import defaultdict
class DomainCounter(object):
def __init__(self):
self.base_path = '/opt/mail_log'
self.tmp = []
self.date = datetime.date.today() - datetime.timedelta(days=14)
self.file_out = '/var/tmp/parsed_exim_files-' + str(self.date.strftime('%Y%m%d')) + '.decompressed'
def parse_log_files(self):
sub_dir = os.listdir(self.base_path)
for directory in sub_dir:
if re.search('smtp\d+', directory):
fileInput = self.base_path + '/' + directory + '/maillog-' + str(self.date.strftime('%Y%m%d')) + '.bz2'
if not os.path.isfile(self.file_out):
os.popen('touch ' + self.file_out)
proccessFiles = os.popen('/bin/bunzip2 -cd ' + fileInput + ' > ' + self.file_out)
accessFileHandle = open(self.file_out, 'r')
readFileHandle = accessFileHandle.readlines()
print "Proccessing %s." % fileInput
for line in readFileHandle:
if '<=' in line and ' for ' in line and '<>' not in line:
distinctLine = line.split(' for ')
recipientAddresses = distinctLine[1].strip()
recipientAddressList = recipientAddresses.strip().split(' ')
if len(recipientAddressList) > 1:
for emailaddress in recipientAddressList:
# Since syslog messages are transmitted over UDP some messages are dropped and needs to be filtered out.
if '#' in emailaddress:
(login, domein) = emailaddress.split("#")
self.tmp.append(domein)
continue
else:
try:
(login, domein) = recipientAddressList[0].split("#")
self.tmp.append(domein)
except Exception as e:
print e, '<<No valid email address found, skipping line>>'
accessFileHandle.close()
os.unlink(self.file_out)
return self.tmp
if __name__ == '__main__':
domainCounter = DomainCounter()
result = domainCounter.parse_log_files()
domainCounts = defaultdict(int)
top = 100
for domain in result:
domainCounts[domain] += 1
sortedDict = dict(sorted(domainCounts.items(), key=lambda x: x[1], reverse=True)[:int(top)])
for w in sorted(sortedDict, key=sortedDict.get, reverse=True):
print '%-3s %s' % (sortedDict[w], w)

proccessFiles = os.popen('/bin/bunzip2 -cd ' + fileInput + ' > ' + self.file_out)
This line is non-blocking. Therefore it will start the command, but the few following lines are already reading the file. This is basically a concurrency issue. Try to wait for the command to complete before reading the file.
Also see:
Python popen command. Wait until the command is finished since os.popen is deprecated since python-2.6 (depending on which version you are using).
Sidenote - The same happens to the line below. The file may, or may not, exist after executing the following line:
os.popen('touch ' + self.file_out)