We Keep Coding

How to check the windows path matches with partial Linux path string

I am trying to check what files that are present in my full_list_files are also present in required_list.
The thing here is they are not exactly equal to one other , but macthes with filename and last sub directory.
Example :
'C:\Users\Documents\Updated\Build\Output\M\Application_1.bin' matches with "M/Application_1.bin" except the slashes are different.
So I am trying to make both uniform by using the function convert_fslash_2_bslash
But still, I see the output as below ,none of the files are matched.
full_list_files = set(['C:\\Users\\Documents\\Updated\\Build\\Output\\O\\Report.tar.gz', 'C:\\Users\\Documents\\Updated\\Build\\Output\\N\\Application_2.bin', 'C:\\Users\\Documents\\Updated\\Build\\Output\\O\\Testing.txt', 'C:\\Users\\Documents\\Updated\\Build\\Output\\M\\masking.tar.gz', 'C:\\Users\\Documents\\Updated\\Build\\Output\\N\\Application_1.bin', 'C:\\Users\\Documents\\Updated\\Build\\Output\\M\\Application_1.bin', 'C:\\Users\\Documents\\Updated\\Build\\Output\\O\\History.zip', 'C:\\Users\\Documents\\Updated\\Build\\Output\\O\\Challenge.tar.gz', 'C:\\Users\\Documents\\Updated\\Build\\Output\\M\\Application_2.bin', 'C:\\Users\\Documents\\Updated\\Build\\Output\\N\\porting.tar.gz', 'C:\\Users\\Documents\\Updated\\Build\\Output\\M\\Booting.tar.gz'])
original required_list = set(['N/Application_2.bin', 'M/masking.tar.gz', 'N/Application_1.bin', 'O/Challenge.tar.gz', 'M/Application_1.bin', 'O/Testing.txt', 'M/rooting.tar.gz', 'M/Application_2.bin', 'O/History.zip', 'N/porting.tar.gz', 'O/Report.tar.gz'])
modified required_list = ['N\\Application_2.bin', 'M\\masking.tar.gz', 'N\\Application_1.bin', 'O\\Challenge.tar.gz', 'M\\Application_1.bin', 'O\\Testing.txt', 'M\\rooting.tar.gz', 'M\\Application_2.bin', 'O\\History.zip', 'N\\porting.tar.gz', 'O\\Report.tar.gz']
'C:\\Users\\Documents\\Updated\\Build\\Output\\O\\Report.tar.gz' not present
'C:\\Users\\Documents\\Updated\\Build\\Output\\N\\Application_2.bin' not present
'C:\\Users\\Documents\\Updated\\Build\\Output\\O\\Testing.txt' not present
'C:\\Users\\Documents\\Updated\\Build\\Output\\M\\masking.tar.gz' not present
'C:\\Users\\Documents\\Updated\\Build\\Output\\N\\Application_1.bin' not present
'C:\\Users\\Documents\\Updated\\Build\\Output\\M\\Application_1.bin' not present
'C:\\Users\\Documents\\Updated\\Build\\Output\\O\\History.zip' not present
'C:\\Users\\Documents\\Updated\\Build\\Output\\O\\Challenge.tar.gz' not present
'C:\\Users\\Documents\\Updated\\Build\\Output\\M\\Application_2.bin' not present
'C:\\Users\\Documents\\Updated\\Build\\Output\\N\\porting.tar.gz' not present
'C:\\Users\\Documents\\Updated\\Build\\Output\\M\\Booting.tar.gz' not present
How can I get it working correctly.
import os
import sys
import re
full_list_files = {
#These are actually real paths parsed from listdir
#Just for convenience used as strings
'C:\Users\Documents\Updated\Build\Output\M\Application_1.bin',
'C:\Users\Documents\Updated\Build\Output\M\Application_2.bin',
'C:\Users\Documents\Updated\Build\Output\M\masking.tar.gz',
'C:\Users\Documents\Updated\Build\Output\M\Booting.tar.gz',
'C:\Users\Documents\Updated\Build\Output\N\Application_1.bin',
'C:\Users\Documents\Updated\Build\Output\N\Application_2.bin',
'C:\Users\Documents\Updated\Build\Output\N\porting.tar.gz',
'C:\Users\Documents\Updated\Build\Output\O\Challenge.tar.gz',
'C:\Users\Documents\Updated\Build\Output\O\History.zip',
'C:\Users\Documents\Updated\Build\Output\O\Testing.txt',
'C:\Users\Documents\Updated\Build\Output\O\Report.tar.gz'
}
required_list = {
"M/Application_1.bin",
"M/Application_2.bin",
"M/masking.tar.gz",
"M/rooting.tar.gz",
"N/Application_1.bin",
"N/Application_2.bin",
"N/porting.tar.gz",
"O/Challenge.tar.gz",
"O/History.zip",
"O/Testing.txt",
"O/Report.tar.gz"
}
def convert_fslash_2_bslash(required_file_list):
required_config_file_list = []
i = 0
for entry in required_file_list:
entry = entry.strip()
entry = entry.replace('"',"")
entry = entry.replace('/','\\')
required_config_file_list.insert(i, entry)
i = i + 1
return required_config_file_list
if __name__ == "__main__":
print
print "full_list_files = ", full_list_files
print
print "original required_list = ", required_list
print
required_config_file_list = convert_fslash_2_bslash(required_list)
print "modified required_list = ", required_config_file_list
print
for f_entry in full_list_files:
f_entry = repr(f_entry)
#for r_entry in required_config_file_list:
#if ( f_entry.find(r_entry) != -1):
if f_entry in required_config_file_list:
print f_entry ," present"
else:
print f_entry ," not present"

Here is the logic you need at the bottom:
for f_entry in full_list_files:
for r_entry in required_config_file_list:
if f_entry.endswith(r_entry):
print f_entry, " present"
You need to loop over both collections, then check to see if the longer path ends with the shorter path. One of your mistakes was calling repr(), which changes the double backslashes to quadruple ones.
I'll leave it up to you to decide how you'll handle printing paths that are not present at all.

Visual Studio: unresolved import 'numpy'

I am trying to run the code below which requires numpy. I installed it via pip install numpy. However, numpy gets highlighted in the editor with the note unresolved import 'numpy'. When I try to run it I get the error No module named 'numpy'. After I got the error the first time I uninstalled numpy and re-installed it but the problem persists.
I am using Python 3.7.8 and NumPy 1.20.2.
The code I am trying to run:
#!/usr/bin/env python3
#
# Copyright (c) 2018 Matthew Earl
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
# USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
Super Mario Bros level extractor
This script requires py65emu, numpy, and PIL to run. Run with no arguments to see usage.
See http://matthewearl.github.io/2018/06/28/smb-level-extractor/ for a description of how this was written.
To run you'll need to compile https://gist.github.com/1wErt3r/4048722 with x816 to obtain the PRG-ROM and symbol files.
The CHR-ROM should be extracted from a Super Mario Bros ROM, or can be read from an INES ROM file. See
https://wiki.nesdev.com/w/index.php/INES for information on the INES format. In addition you'll need a NES palette
saved in "data/ntscpalette.pal", generated using the tool here: https://bisqwit.iki.fi/utils/nespalette.php
"""
import collections
import pathlib
import re
import numpy as np
from py65emu.cpu import CPU
from py65emu.mmu import MMU
_WORKING_RAM_SIZE = 0x800
Symbol = collections.namedtuple('Symbol', ('name', 'address', 'line_num'))
class SymbolFile:
_LINE_RE = r"(?P<name>[A-Z0-9_]+) *= \$(?P<address>[A-F0-9]*) *; <> \d+, statement #(?P<line_num>\d+)"
def __init__(self, fname):
with open(fname) as f:
self._symbols = [self._parse_symbol(line) for line in f.readlines()]
self._symbols = list(sorted(self._symbols, key=lambda s: s.address))
self._name_to_addr = {s.name: s.address for s in self._symbols}
self._addr_to_name = {s.address: s.name for s in self._symbols}
def _parse_symbol(self, line):
m = re.match(self._LINE_RE, line)
return Symbol(m.group('name'), int(m.group('address'), 16), int(m.group('line_num')))
def __getitem__(self, name):
return self._name_to_addr[name]
def _read_ppu_data(mmu, addr):
while True:
ppu_high_addr = mmu.read(addr)
if ppu_high_addr == 0x0:
break
ppu_low_addr = mmu.read(addr + 1)
assert ppu_high_addr == 0x3f and ppu_low_addr == 0x00
flags_and_length = mmu.read(addr + 2)
assert (flags_and_length & (1<<7)) == 0, "32-byte increment flag set"
assert (flags_and_length & (1<<6)) == 0, "Repeating flag set"
length = flags_and_length & 0b111111
addr += 3
for i in range(length):
yield mmu.read(addr)
addr += 1
def _load_palette(mmu, sym_file, nes_palette):
area_type = mmu.read(sym_file['AREATYPE'])
idx = mmu.read(sym_file['AREAPALETTE'] + area_type)
high_addr = mmu.read(sym_file['VRAM_ADDRTABLE_HIGH'] + idx)
low_addr = mmu.read(sym_file['VRAM_ADDRTABLE_LOW'] + idx)
palette_data = list(_read_ppu_data(mmu, high_addr << 8 | low_addr))
assert len(palette_data) == 32
a = np.array(palette_data[:16]).reshape(4, 4)
a[:, 0] = mmu.read(sym_file['BACKGROUNDCOLORS'] + area_type)
return nes_palette[a]
def _execute_subroutine(cpu, addr):
s_before = cpu.r.s
cpu.JSR(addr)
while cpu.r.s != s_before:
cpu.step()
def _get_metatile_buffer(mmu, sym_file):
return [mmu.read(sym_file['METATILEBUFFER'] + i) for i in range(13)]
def load_tile(chr_rom, idx):
chr_rom_addr = 0x1000 + 16 * idx
d = chr_rom[chr_rom_addr:chr_rom_addr + 16]
a = np.array([[b & (128 >> i) != 0 for i in range(8)] for b in d]).reshape(2, 8, 8)
return a[0] + 2 * a[1]
def _render_metatile(mmu, chr_rom, mtile, palette):
palette_num = mtile >> 6
palette_idx = mtile & 0b111111
high_addr = mmu.read(sym_file['METATILEGRAPHICS_HIGH'] + palette_num)
low_addr = mmu.read(sym_file['METATILEGRAPHICS_LOW'] + palette_num)
addr = (high_addr << 8 | low_addr) + palette_idx * 4
t = np.vstack([np.hstack([load_tile(chr_rom, mmu.read(addr + c * 2 + r)) for c in range(2)])
for r in range(2)])
return palette[palette_num][t]
def load_level(stage, prg_rom, chr_rom, sym_file, nes_palette):
# Initialize the MMU / CPU
mmu = MMU([
(0x0, _WORKING_RAM_SIZE, False, []),
(0x8000, 0x10000, True, list(prg_rom))
])
cpu = CPU(mmu, 0x0)
# Execute some preamble subroutines which set up variables used by the main subroutines.
if isinstance(stage, tuple):
world_num, area_num = stage
mmu.write(sym_file['WORLDNUMBER'], world_num - 1)
mmu.write(sym_file['AREANUMBER'], area_num - 1)
_execute_subroutine(cpu, sym_file['LOADAREAPOINTER'])
else:
area_pointer = stage
mmu.write(sym_file['AREAPOINTER'], area_pointer)
mmu.write(sym_file['HALFWAYPAGE'], 0)
mmu.write(sym_file['ALTENTRANCECONTROL'], 0)
mmu.write(sym_file['PRIMARYHARDMODE'], 0)
mmu.write(sym_file['OPERMODE_TASK'], 0)
_execute_subroutine(cpu, sym_file['INITIALIZEAREA'])
# Extract the palette.
palette = _load_palette(mmu, sym_file, nes_palette)
# Repeatedly extract meta-tile columns, until the level starts repeating.
cols = []
for column_pos in range(1000):
_execute_subroutine(cpu, sym_file['AREAPARSERCORE'])
cols.append(_get_metatile_buffer(mmu, sym_file))
_execute_subroutine(cpu, sym_file['INCREMENTCOLUMNPOS'])
if len(cols) >= 96 and cols[-48:] == cols[-96:-48]:
cols = cols[:-80]
break
level = np.array(cols).T
# Render a dict of metatiles.
mtiles = {mtile: _render_metatile(mmu, chr_rom, mtile, palette)
for mtile in set(level.flatten())}
return level, mtiles
def render_level(level, mtiles):
return np.vstack([np.hstack([mtiles[mtile] for mtile in row]) for row in level])
if __name__ == "__main__":
import sys
import PIL.Image
world_map = {
'{}-{}'.format(world_num, area_num): (world_num, area_num)
for world_num in range(1, 9)
for area_num in range(1, 5)
}
world_map.update({
'bonus': 0xc2,
'cloud1': 0x2b,
'cloud2': 0x34,
'water1': 0x00,
'water2': 0x02,
'warp': 0x2f,
})
if len(sys.argv) < 6:
print("Usage: {} <world> <prg-rom> <sym-file> <chr-rom> <out-file>".format(sys.argv[0]), file=sys.stderr)
print(" <world> is one of {}".format(', '.join(sorted(world_map.keys()))), file=sys.stderr)
print(" <prg-rom> is the binary output from x816")
print(" <sym-file> is the sym file output from x816")
print(" <chr-rom> is a CHR-ROM dump")
print(" <out-file> is the output image name")
sys.exit(-1)
stage = world_map[sys.argv[1]]
with open(sys.argv[2], 'rb') as f:
prg_rom = f.read()
sym_file = SymbolFile(sys.argv[3])
with open(sys.argv[4], 'rb') as f:
chr_rom = f.read()
out_fname = sys.argv[5]
with (pathlib.Path(sys.argv[0]).parent / "data" / "ntscpalette.pal").open("rb") as f:
nes_palette = np.array(list(f.read())).reshape(64, 3)
level, mtiles = load_level(stage, prg_rom, chr_rom, sym_file, nes_palette)
a = render_level(level, mtiles).astype(np.uint8)
im = PIL.Image.fromarray(a)
im.save(out_fname)

How did you create your workspace in Visual Studio? Do you have Python development tools installed with Visual Studio? Did you create a "Python application" as your project template?
If so then your project should have a virtual environment created, which you can see in the solution directory. If that is the case do:
Go to "Solution Explorer" Tab >
Find "Python Environments"
Find your active env. For me there was only one called "Python 3.9 (global default)"
Right click and select "Manage Python Packages..."
There it should list all the packages installed and versions. If numpy is not there, just type "numpy" in the search box and click the suggests install option: "run command: pip install numpy".

Make sure you have installed NumPy in the same python environment that you use to run the program. (Check the PATH variable if it includes the path to the correct python environment)

Get the latest FTP folder name in Python

I am trying to write a script to get the latest file from the latest sub-
directory of FTP server in Python. My problem is I am unable to figure out the
latest sub-directory. There are two options available, sub-directories have ctime available. Also in directory name date is mentioned that on which date directory was created. But I do not know how to get the name of the latest directory. I have figured out the following way (hoping for the server side to be sorted by latest ctime). I have done it the following way which will work if first object is the latest directory.
import ftplib
import os
import time
ftp = ftplib.FTP('test.rebex.net','demo', 'password')
ftp.cwd(str((ftp.nlst())[0])) #if directory is sorted in descending order by date.
But is there any way where I will find the exact directory by ctime or by date in directory name ?
Thanks a lot guys.

If your FTP server supports MLSD command, a solution is easy:
If you want to base the decision on a modification timestamp:
entries = list(ftp.mlsd())
# Only interested in directories
entries = [entry for entry in entries if entry[1]["type"] == "dir"]
# Sort by timestamp
entries.sort(key = lambda entry: entry[1]['modify'], reverse = True)
# Pick the first one
latest_name = entries[0][0]
print(latest_name)
If you want to use a file name:
# Sort by filename
entries.sort(key = lambda entry: entry[0], reverse = True)
If you need to rely on an obsolete LIST command, you have to parse a proprietary listing it returns.
A common *nix listing is like:
drw-r--r-- 1 user group 4096 Mar 26 2018 folder1-20180326
drw-r--r-- 1 user group 4096 Jun 18 11:21 folder2-20180618
-rw-r--r-- 1 user group 4467 Mar 27 2018 file-20180327.zip
-rw-r--r-- 1 user group 124529 Jun 18 15:31 file-20180618.zip
With a listing like this, this code will do:
If you want to base the decision on a modification timestamp:
lines = []
ftp.dir("", lines.append)
latest_time = None
latest_name = None
for line in lines:
tokens = line.split(maxsplit = 9)
# Only interested in directories
if tokens[0][0] == "d":
time_str = tokens[5] + " " + tokens[6] + " " + tokens[7]
time = parser.parse(time_str)
if (latest_time is None) or (time > latest_time):
latest_name = tokens[8]
latest_time = time
print(latest_name)
If you want to use a file name:
lines = []
ftp.dir("", lines.append)
latest_name = None
for line in lines:
tokens = line.split(maxsplit = 9)
# Only interested in directories
if tokens[0][0] == "d":
name = tokens[8]
if (latest_name is None) or (name > latest_name):
latest_name = name
print(latest_name)
Some FTP servers may return . and .. entries in LIST results. You may need to filter those.
Partially based on: Python FTP get the most recent file by date.
If the folder does not contain any files, only subfolders, there are other easier options.
If you want to base the decision on a modification timestamp and the server supports non-standard -t switch, you can use:
lines = ftp.nlst("-t")
latest_name = lines[-1]
See How to get files in FTP folder sorted by modification time
If you want to use a file name:
lines = ftp.nlst()
latest_name = max(lines)

py2exe SytaxError: invalid syntax (asyncsupport.py, line22) [duplicate]

This command works fine on my personal computer but keeps giving me this error on my work PC. What could be going on? I can run the Char_Limits.py script directly in Powershell without a problem.
error: compiling 'C:\ProgramData\Anaconda2\lib\site-packages\jinja2\asyncsupport.py' failed
SyntaxError: invalid syntax (asyncsupport.py, line 22)
My setup.py file looks like:
from distutils.core import setup
import py2exe
setup (console=['Char_Limits.py'])
My file looks like:
import xlwings as xw
from win32com.client import constants as c
import win32api
"""
Important Notes: Header row has to be the first row. No columns without a header row. If you need/want a blank column, just place a random placeholder
header value in the first row.
Product_Article_Number column is used to determine the number of rows. It must be populated for every row.
"""
#functions, hooray!
def setRange(columnDict, columnHeader):
column = columnDict[columnHeader]
rngForFormatting = xw.Range((2,column), (bttm, column))
cellReference = xw.Range((2,column)).get_address(False, False)
return rngForFormatting, cellReference
def msg_box(message):
win32api.MessageBox(wb.app.hwnd, message)
#Character limits for fields in Hybris
CharLimits_Fields = {"alerts":500, "certifications":255, "productTitle":300,
"teaserText":450 , "includes":1000, "compliance":255, "disclaimers":9000,
"ecommDescription100":100, "ecommDescription240":240,
"internalKeyword":1000, "metaKeywords":1000, "metaDescription":1000,
"productFeatures":7500, "productLongDescription":1500,"requires":500,
"servicePlan":255, "skuDifferentiatorText":255, "storage":255,
"techDetailsAndRefs":12000, "warranty":1000}
# Fields for which a break tag is problematic.
BreakTagNotAllowed = ["ecommDescription100", "ecommDescription240", "productTitle",
"skuDifferentiatorText"]
app = xw.apps.active
wb = xw.Book(r'C:\Users\XXXX\Documents\Import File.xlsx')
#identifies the blanket range of interest
firstCell = xw.Range('A1')
lstcolumn = firstCell.end("right").column
headers_Row = xw.Range((1,1), (1, lstcolumn)).value
columnDict = {}
for column in range(1, len(headers_Row) + 1):
header = headers_Row[column - 1]
columnDict[header] = column
try:
articleColumn = columnDict["Product_Article_Number"]
except:
articleColumn = columnDict["Family_Article_Number"]
firstCell = xw.Range((1,articleColumn))
bttm = firstCell.end("down").row
wholeRange = xw.Range((1,1),(bttm, lstcolumn))
wholeRangeVal = wholeRange.value
#Sets the font and deletes previous conditional formatting
wholeRange.api.Font.Name = "Arial Unicode MS"
wholeRange.api.FormatConditions.Delete()
for columnHeader in columnDict.keys():
if columnHeader in CharLimits_Fields.keys():
rng, cellRef = setRange(columnDict, columnHeader)
rng.api.FormatConditions.Add(2,3, "=len(" + cellRef + ") >=" + str(CharLimits_Fields[columnHeader]))
rng.api.FormatConditions(1).Interior.ColorIndex = 3
if columnHeader in BreakTagNotAllowed:
rng, cellRef = setRange(columnDict, columnHeader)
rng.api.FormatConditions.Add(2,3, '=OR(ISNUMBER(SEARCH("<br>",' + cellRef + ')), ISNUMBER(SEARCH("<br/>",' + cellRef + ")))")
rng.api.FormatConditions(2).Interior.ColorIndex = 6
searchResults = wholeRange.api.Find("~\"")
if searchResults is not None:
msg_box("There's a double quote in this spreadsheet")
else:
msg_box("There are no double quotes in this spreadsheet")
# app.api.FindFormat.Clear
# app.api.FindFormat.Interior.ColorIndex = 3
# foundRed = wholeRange.api.Find("*", SearchFormat=True)
# if foundRed is None:
# msg_box("There are no values exceeding character limits")
# else:
# msg_box("There are values exceeding character limits")
# app.api.FindFormat.Clear
# app.api.FindFormat.Interior.ColorIndex = 6
# foundYellow = wholeRange.api.Find("*", SearchFormat=True)
# if foundYellow is None:
# msg_box("There are no break tags in this spreadsheet")
# else:
# msg_box("There are break tags in this spreadsheet")

Note:
If you are reading this, I would try Santiago's solution first.
The issue:
Looking at what is likely at line 22 on the github package:
async def concat_async(async_gen):
This is making use of the async keyword which was added in python 3.5, however py2exe only supports up to python 3.4. Now jinja looks to be extending the python language in some way (perhaps during runtime?) to support this async keyword in earlier versions of python. py2exe cannot account for this language extension.
The Fix:
async support was added in jinja2 version 2.9 according to the documentation. So I tried installing an earlier version of jinja (version 2.8) which I downloaded here.
I made a backup of my current jinja installation by moving the contents of %PYTHONHOME%\Lib\site-packages\jinja2 to some other place.
extract the previously downloaded tar.gz file and install the package via pip:
cd .\Downloads\dist\Jinja2-2.8 # or wherever you extracted jinja2.8
python setup.py install
As a side note, I also had to increase my recursion limit because py2exe was reaching the default limit.
from distutils.core import setup
import py2exe
import sys
sys.setrecursionlimit(5000)
setup (console=['test.py'])
Warning:
If whatever it is you are using relies on the latest version of jinja2, then this might fail or have unintended side effects when actually running your code. I was compiling a very simple script.

I had the same trouble coding in python3.7. I fixed that adding the excludes part to my py2exe file:
a = Analysis(['pyinst_test.py'],
#...
excludes=['jinja2.asyncsupport','jinja2.asyncfilters'],
#...)
I took that from: https://github.com/pyinstaller/pyinstaller/issues/2393

Python script to alert on empty/missing logs

I am working on a project to check a file directory and automatically add log files as they are created. A file is being generated every five minutes, but some of the files are being created with a "0" filesize and I would like to alert when this happens.
So the sequence of steps I would like to have are essentially:
Get time (MM:DD:YY HH:MM:SS) *Not sure if I need to do this...
CD to Folder Directory /Netflow/YY/MM/DD
Search for filename "nfcapd.YYYYMMDDHHMM" where MM increments by 5.
If filesize is 0, then email Johnny, Sally and Jimmy
Wait 6 minutes and repeat
This is what I have pieced together thus far. How can I get the desired functionality?
import os
def is_non_zero_file(fpath): storage/Netflow/
return True if os.path.isfile(fpath) and os.path.getsize(fpath) > 0 else False
# I need to check storage/Netflow for files named by time e.g 13_56_05.txt
while True:
time.sleep(360)

In addition to enumerating the files in a given path, and subsequently filtering the files which are only zero-length, you probably want to maintain some type of state to ensure you're aren't notified multiple times of the same zero length file. That is, you probably don't want to get a notification that the same file is zero-length indefinitely (although you can modify the example below if you want said behavior).
You may optionally want to do things like verify that the file name strictly meets your naming convention. You may also want to validate the the string date-stamp included in the file name is a valid datetime.
The example below uses the glob module (itself leveraging os.listdir() and fnmatch.fnmatch()) to build up a set of possible files for inclusion. [1]
The example is intentionally simple, and leverages a single class to store log sample 'state'. KEEP_SAMPLES samples are maintained (instances of logState() in the log_states list, achieved by using list slicing.
A single alert(msg) function is supplied as a stub to something that might send mail, etc...
References:
[1] https://docs.python.org/3.2/library/glob.html
#!/usr/bin/python3
import os
import glob
import re
from datetime import datetime, timezone
import time
from pprint import pprint
class logState():
def __init__(self, log_path, glob_patt, re_patt, dt_fmt):
self.dt = datetime.now(timezone.utc)
self.log_path = log_path
self.glob_patt = glob_patt
self.re_patt = re_patt
self.dt_fmt = dt_fmt
self.empty_logs = []
self.nonempty_logs = []
# Retrieve only files from glob
self.files = [ f for f in
glob.glob(self.log_path + self.glob_patt)
if os.path.isfile(f) ]
for f in self.files:
unq_fname = f.split('/')[-1]
if unq_fname == None:
continue
# Tighter pattern matching
if re.match(re_patt, unq_fname) == None:
continue
# Get the datetime portion of the file name
f_dtstamp = unq_fname.split('.')[-1]
# Make sure the datetime stamp represents
# a valid date
if datetime.strptime(f_dtstamp, self.dt_fmt) == None:
continue
# Check file size, add to the appropriate
# list
if os.path.getsize(f) <= 0:
self.empty_logs.append(f)
else:
self.nonempty_logs.append(f)
def alert(msg):
print("ALERT!: {0}".format(msg))
if __name__ == "__main__":
# How long to sleep
SLEEP_SECS = 5
# How many samples to keep
KEEP_SAMPLES = 5
log_states = []
# Definition for what logs states we'll look for
log_path = './'
glob_patt = 'nfcapd.[0-9]*'
re_patt = 'nfcapd.([0-9]{12})'
dt_fmt = "%Y%m%d%H%M"
print("-- Setup --")
print("Sample files in '{0}'".format(log_path))
print("\t{0} samples kept:".format(KEEP_SAMPLES))
print("\tglob pattern: '{0}'".format(glob_patt))
print("\tregex pattern: '{0}'".format(re_patt))
print("\tdatetime string: '{0}'".format(dt_fmt))
print("")
# Collect the initial state
log_states.append(logState(log_path,
glob_patt,
re_patt, dt_fmt))
while True:
# Print state inventory and current state detail
print( "-- Log States Stored --")
for i, log_state in enumerate(log_states):
print("Log state {0} # {1}".format(i, log_state.dt))
print(" -- Logs size > 0 --")
pprint(log_states[-1].nonempty_logs)
print(" -- Logs size <= 0 --")
pprint(log_states[-1].empty_logs)
print("")
time.sleep(SLEEP_SECS)
log_states = log_states[-KEEP_SAMPLES+1:]
log_states.append(logState(log_path,
glob_patt,
re_patt,
dt_fmt))
# p = previous sample, c = current
p = set(log_states[-2].empty_logs)
c = set(log_states[-1].empty_logs)
# only report the items in the current sample
# not in the last
if len(c.difference(p)) > 0:
alert("\nNew zero length logs: " + str(c.difference(p)) + "\n")