We Keep Coding

File path artifacts when reading target path from a shortcut(.lnk) in python?

When testing the code from this answer, I do receive an output, however it seems to have some artifacts...
This is the code I tested:
import locale
import struct
def __readLink(path):
target = ''
try:
with open(path, 'rb') as stream:
content = stream.read()
# skip first 20 bytes (HeaderSize and LinkCLSID)
# read the LinkFlags structure (4 bytes)
lflags = struct.unpack('I', content[0x14:0x18])[0]
position = 0x18
# if the HasLinkTargetIDList bit is set then skip the stored IDList
# structure and header
if (lflags & 0x01) == 1:
position = struct.unpack('H', content[0x4C:0x4E])[0] + 0x4E
last_pos = position
position += 0x04
# get how long the file information is (LinkInfoSize)
length = struct.unpack('I', content[last_pos:position])[0]
# skip 12 bytes (LinkInfoHeaderSize, LinkInfoFlags, and VolumeIDOffset)
position += 0x0C
# go to the LocalBasePath position
lbpos = struct.unpack('I', content[position:position+0x04])[0]
position = last_pos + lbpos
# read the string at the given position of the determined length
size= (length + last_pos) - position - 0x02
temp = struct.unpack('c' * size, content[position:position+size])
target = ''.join([chr(ord(a)) for a in temp])
except:
# could not read the file
pass
return target
print(__readLink('test.lnk'))
The output is linked below, as for some reason, it does not copy completely.
Another problem I see is that it does not output the full file extension? It should be an .mp4, but it terminates at ".mp"
Image of output

The Code from the Question does not follow the MS-SHLLINK specification,
there are to many fixed Offset Values.
MS-SHLLINK: Shell Link (.LNK) Binary File Format
Specifies the Shell Link Binary File Format, which contains information that can be used to access another data object. The Shell Link Binary File Format is the format of Windows files with the extension "LNK".
The following code uses only LinkFlags 0x14 and LinkTargetIDList 0x4c as fixed Offsets.
def LocalBasePath(path):
def unpack(offset, size):
m = 'I'
if size == 2: m = 'H'
return struct.unpack(m, content[offset:offset+size])[0]
target = ''
try:
with open(path, 'rb') as fh:
content = fh.read()
# Read the LinkFlags 4 bytes
LinkFlags = unpack(0x14, 4)
position = 0x4c
# Skip LinkTargetIDList if HasLinkTargetIDList
if (LinkFlags & 0x01) == 1:
position += unpack(position, 2)
position += 0x02 # TerminalID 2 bytes
LinkInfo_offset = position
LinkInfoSize = unpack(position, 4)
# Skip 4 * 4 bytes in LinkInfo
LokalBasePathOffset = LinkInfo_offset + (4 * 4)
LocalBasePath = LinkInfo_offset + unpack(LokalBasePathOffset, 4)
# Read LocalBasePath String
size = ((LinkInfo_offset + LinkInfoSize) - LocalBasePath) -2
target = ''.join([chr(ord(a)) for a in struct.unpack('c' * size, content[LocalBasePath:LocalBasePath+size])])
except Exception as exp:
print(exp)
return target
Tested with Python: 3.4.2

Converting Binary to ASCII, and ASCII to Binary

I'm currently writing an ascii-binary/binary-ascii converter in Python for a school project, and I have an issue with converting from ascii (String text) to binary. The idea is to print the outcome in the test() on the bottom of the code.
When running the code in WingIDE, an error occurs:
On the line starting with
bnary = bnary + binary[chnk]
KeyError: "Norway stun Poland 30:28 and spoil Bielecki's birthday party."
What I'm trying to do here is to convert the String of text stored in "text.txt" to a String of integers, and then print this binary string.
Any help is greatly appreciated. I tried looking at other ascii-binary vice-versa convertion related questions, but none seemed to work for me.
My code:
def code():
binary = {}
ascii = {}
# Generate ascii code
for i in range(0,128) :
ascii[format(i,'08b')] = chr(i)
# Reverse the ascii code, this will be binary
for k, v in ascii.iteritems():
binary[v] = binary.get(v, [])
binary[v].append(k)
return ascii
def encode(text,binary):
'''
Encode some text using text from a source
'''
bnary = ""
fi = open(text, mode='rb')
while True:
chnk = fi.read()
if chnk == '':
break
if chnk != '\n':
binry = ""
bnary = bnary + binary[chnk]
return bnary
def decode(sourcecode,n, ascii):
'''
Decode a sourcecode using chunks of size n
'''
sentence = ""
f = open(sourcecode, mode='rb') # Open a file with filename <sourcecode>
while True:
chunk = f.read(n) # Read n characters at time from an open file
if chunk == '': # This is one way to check for the End Of File in Python
break
if chunk != '\n':
setence = "" # The ascii sentence generated
# create a sentence
sentence = sentence + ascii[chunk]
return sentence
def test():
'''
A placeholder for some test cases.
It is recommended that you use some existing framework, like unittest,
but for a temporary testing in a development version can be done
directly in the module.
'''
print encode('text.txt', code())
print decode('sourcecode.txt', 8, code())
test()

If you want to decode and encode, have this solutions
Encode ascii to bin
def toBinary(string):
return "".join([format(ord(char),'#010b')[2:] for char in string])
Encode bin to ascii
def toString(binaryString):
return "".join([chr(int(binaryString[i:i+8],2)) for i in range(0,len(binaryString),8)])

fi.read() returns the whole document the first time and '' next times. So you should do
text = fi.read()
for char in text:
do_stuff()
Edit1
You can only read your file once. Thus you have to get your chars one by one. A file.read returns a string containing the whole document.
You can iterate over a string to get chars one by one.
The main error is your binary is {"a":['010001110'], "b":...} and you try to access with the key "azerty" where you should do it char by char:
string = "azer"
result = []
for c in string:
result += binary[c]
>>> result = [['11001'],[1001101'],...]

# Lets say your filename is stored in fname
def binary(n):
return '{0:08b}'.format(n)
with open(fname) as f:
content = f.readlines()
for i in content:
print(binary(ord(i)), end='')
print('')
This will give you the integer value(from ascii) of each character in the file, line by line

Getting the memory layout out of an (avr)elf file by useing python + pyElftools

I am creating my own bootloader for an ATXmega128A4U. To use the bootloader I want to transform the ELF-file of the firmware into a memory map used in the the ATXmega.
For that I use python and the modul "pyelftools". The documentation of it is poor and so I run into a problem: I do not know what information I can use to get the address, offset etc. from the data at the sections.
My goal is to create a bytearray, copy the data/code into it and transfer it to the bootlaoder. Below is my code:
import sys
# If pyelftools is not installed, the example can also run from the root or
# examples/ dir of the source distribution.
sys.path[0:0] = ['.', '..']
from elftools.common.py3compat import bytes2str
from elftools.elf.elffile import ELFFile
# 128k flash for the ATXmega128a4u
flashsize = 128 * 1024
def process_file(filename):
with open(filename, 'rb') as f:
# get the data
elffile = ELFFile(f)
dataSec = elffile.get_section_by_name(b'.data')
textSec = elffile.get_section_by_name(b'.text')
# prepare the memory
flashMemory = bytearray(flashsize)
# the data section
startAddr = dataSec.header.sh_offset
am = dataSec.header.sh_size
i = 0
while i < am:
val = dataSec.stream.read(1)
flashMemory[startAddr] = val[0]
startAddr += 1
i += 1
# the text section
startAddr = textSec.header.sh_offset
am = textSec.header.sh_size
i = 0
while i < am:
print(str(startAddr) + ' : ' + str(i))
val = textSec.stream.read(1)
flashMemory[startAddr] = val[0]
startAddr += 1
i += 1
print('finished')
if __name__ == '__main__':
process_file('firmware.elf')
Hope someone can tell me how to solve this problem.

I manged to solve the problem.
don't read the data manualy from the stream by "textSec.stream.read" use "textSec.data()" instead. Internaly (see "sections.py") a seek operation in the file is done. Afterwards the data is read. The result will be the valid data chunk.
The following code reads the code(text) section of a atxmega firmware and copies it into a bytearray which has the layout of the flash of an atxmega128a4u device.
#vlas_tepesch: the hex conversation is not needed and the the 64k pitfall is avoided.
sys.path[0:0] = ['.', '..']
from elftools.common.py3compat import bytes2str
from elftools.elf.elffile import ELFFile
# 128k flash for the ATXmega128a4u
flashsize = 128 * 1024
def __printSectionInfo (s):
print ('[{nr}] {name} {type} {addr} {offs} {size}'.format(
nr = s.header['sh_name'],
name = s.name,
type = s.header['sh_type'],
addr = s.header['sh_addr'],
offs = s.header['sh_offset'],
size = s.header['sh_size']
)
)
def process_file(filename):
print('In file: ' + filename)
with open(filename, 'rb') as f:
# get the data
elffile = ELFFile(f)
print ('sections:')
for s in elffile.iter_sections():
__printSectionInfo(s)
print ('get the code from the .text section')
textSec = elffile.get_section_by_name(b'.text')
# prepare the memory
flashMemory = bytearray(flashsize)
# the text section
startAddr = textSec.header['sh_addr']
val = textSec.data()
flashMemory[startAddr:startAddr+len(val)] = val
# print memory
print('finished')
if __name__ == '__main__':
process_file('firmware.elf')
Tanks for the comments!

Searching a file for matches between two values and outputting search hits in Python

I am (attempting) to write a program that searches through a hex file for instances of a hex string between two values, eg. Between D4135B and D414AC, incrementing between the first value until the second is reached- D4135B, D4135C, D4135D etc etc.
I have managed to get it to increment etc, but it’s the search part I am having trouble with.
This is the code I have so far, it's been cobbled together from other places and I need to make it somehow output all search hits into the output file (file_out)
I have exceeded the limit of my Python understanding and I'm sure there's probably a much easier way of doing this. I would be very grateful for any help.
def search_process(hx): # searching for two binary strings
global FLAG
while threeByteHexPlusOne != threeByteHex2: #Keep incrementing until second value reached
If Flag:
if hx.find(threeByteHex2) != -1:
FLAG = False #If threeByteHex = ThreeByteHexPlusOne, end search
Print (“Reached the end of the search”,hx.find(threeByteHexPlusOne))
Else:
If hx.find(threeByteHexPlusOne) != -1:
FLAG = True
Return -1 #If no results found
if __name__ == '__main__':
try:
file_in = open(FILE_IN, "r") #opening input file
file_out = open(FILE_OUT, 'w') #opening output file
hx_read = file_in.read #read from input file
tmp = ''
found = ''
while hx_read: #reading from file till file is empty
hx_read = tmp + hx_read
pos = search_process(hx_read)
while pos != -1:
hex_read = hx_read[pos:]
if FLAG:
found = found + hx_read
pos = search_process(hx_read)
tmp = bytes_read[]
hx_read = file_in.read
file_out.write(found) #writing to output file
except IOError:
print('FILE NOT FOUND!!! Check your filename or directory/PATH')

Here's a program that looks through a hex string from a file 3 bytes at a time and if the 3-byte hex string is between the given hex bounds, it writes it to another file. It makes use of generators to make getting the bytes from the hex string a little cleaner.
import base64
import sys
_usage_string = 'Usage: python {} <input_file> <output_file>'.format(sys.argv[0])
def _to_base_10_int(value):
return int(value, 16)
def get_bytes(hex_str):
# Two characters equals one byte
for i in range(0, len(hex_str), 2):
yield hex_str[i:i+2]
def get_three_byte_hexes(hex_str):
bytes = get_bytes(hex_str)
while True:
try:
three_byte_hex = next(bytes) + next(bytes) + next(bytes)
except StopIteration:
break
yield three_byte_hex
def find_hexes_in_range(hex_str, lower_bound_hex, upper_bound_hex):
lower_bound = _to_base_10_int(lower_bound_hex)
upper_bound = _to_base_10_int(upper_bound_hex)
found = []
for three_byte_hex in get_three_byte_hexes(hex_str):
hex_value = _to_base_10_int(three_byte_hex)
if lower_bound <= hex_value < upper_bound:
found.append(three_byte_hex)
return found
if __name__ == "__main__":
try:
assert(len(sys.argv) == 3)
except AssertionError:
print _usage_string
sys.exit(2)
file_contents = open(sys.argv[1], 'rb').read()
hex_str = base64.decodestring(file_contents).encode('hex')
found = find_hexes_in_range(hex_str, 'D4135B', 'D414AC')
print('Found:')
print(found)
if found:
with open(sys.argv[2], 'wb') as fout:
for _hex in found:
fout.write(_hex)
Check out some more info on generators here

Python write to file based on offset

I want to create a python program which splits up a files into segments of specified width, and then a consumer program takes the segments and creates a duplicate of the original file. The segments might be out of order so I intent to use the offset value to write to the file.
Is there a way I can achieve this with without creating a local array to hold all the data on the receiving end?
for example,
f = open(file, "wb")
f.seek(offset)
f.write(data)
The idea behind this is that the program that sends the file might not be able to finish sending the file, and will resume again once it has started.
I have a sample code below which the "combine_bytes" function throws an exception when I try placing data in the buffer location.
import sys
import os
def SplitFile(fname, start, end, width):
t_fileSize = os.path.getsize(fname)
buffData = bytearray(t_fileSize)
for line, offset in get_bytes(fname, int(start), int(end), int(width)):
combine_bytes(buffData, offset, line, width)
nums = ["%02x" % ord(c) for c in line]
print " ".join(nums)
f = open("Green_copy.jpg", "wb")
f.write(buffData)
f.close()
def combine_bytes(in_buff, in_offset, in_data, in_width):
#something like memcpy would be nice
#in_buff[in_offset:in_offset + in_width] = in_data
#this works but it's the mother of inefficiency
i = in_offset
for c in in_data:
in_buff.insert(i, c)
i = i + 1
def get_bytes(fname, start, end, width):
t_currOffset = start
t_width = width
f = open(fname, "r+b")
if end != 0:
while t_currOffset < end:
f.seek(t_currOffset)
if (t_currOffset + t_width) > end:
t_width = end - t_currOffset
t_data = f.read(t_width)
yield t_data,t_currOffset
t_currOffset += t_width
else:
f.seek(t_currOffset)
t_data = f.read(t_width)
while t_data:
yield t_data, t_currOffset
t_currOffset += t_width
f.seek(t_currOffset)
t_data = f.read(t_width)
f.close()
if __name__ == '__main__':
try:
SplitFile(*sys.argv[1:5])
except:
print "Unexpected error:", sys.exc_info()[0]

I still could nt figure out what is your intent - but this version of combine_bytes will get rid of your "mother of your inefficiency" part (which actually is exactly that)
def combine_bytes(in_buff, in_offset, in_data, in_width):
#something like memcpy would be nice
#in_buff[in_offset:in_offset + in_width] = in_data
in_buff = in_buff[:in_offset] + in_data + in_buff[in_offset:]
return in_buff
Of course this creates a new (larger) buffer for each call, and you have to replace your buffer on the caller scope with the one returned:
buffData = combine_bytes(buffData, offset, line, width)

Found it. here is a better way which produces the what I wanted and is faster. _buffData[t_offset:t_offset + len(t_data)] = bytearray(t_data)