I am new to Python. In Perl, to set specific bits to a scalar variable(integer), I can use vec() as below.
#!/usr/bin/perl -w
$vec = '';
vec($vec, 3, 4) = 1; # bits 0 to 3
vec($vec, 7, 4) = 10; # bits 4 to 7
vec($vec, 11, 4) = 3; # bits 8 to 11
vec($vec, 15, 4) = 15; # bits 12 to 15
print("vec() Has a created a string of nybbles,
in hex: ", unpack("h*", $vec), "\n");
Output:
vec() Has a created a string of nybbles,
in hex: 0001000a0003000f
I was wondering how to achieve the same in Python, without having to write bit manipulation code and using struct.pack manually?
Not sure how the vec function works in pearl (haven't worked with the vec function). However, according to the output you have mentioned, the following code in python works fine. I do not see the significance of the second argument. To call the vec function this way: vec(value, size). Every time you do so, the output string will be concatenated to the global final_str variable.
final_vec = ''
def vec(value, size):
global final_vec
prefix = ''
str_hex = str(hex(value)).replace('0x','')
str_hex_size = len(str_hex)
for i in range (0, size - str_hex_size):
prefix = prefix + '0'
str_hex = prefix + str_hex
final_vec = final_vec + str_hex
return 0
vec(1, 4)
vec(10, 4)
vec(3, 4)
vec(15, 4)
print(final_vec)
If you really want to create a hex string from nibbles, you could solve it this way
nibbles = [1,10,3,15]
hex = '0x' + "".join([ "%04x" % x for x in nibbles])
Related
I'm a newbie in this field and am trying to learn a bit about how to write cryptographic hash functions.
To get some hands-on, I tried updating the PySHA2 algorithm for Python 3.6 and up (the original version doesn't work on Python 2.5+ and the author says he won't fix this). I don't intend to use this algorithm for any work, just coding this for the sake of knowledge.
I've reached this far:
import copy
import struct
_initial_hashes = [0x6a09e667f3bcc908, 0xbb67ae8584caa73b, 0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1,
0x510e527fade682d1, 0x9b05688c2b3e6c1f, 0x1f83d9abfb41bd6b, 0x5be0cd19137e2179]
_round_constants = [0x428a2f98d728ae22, 0x7137449123ef65cd, 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc,
0x3956c25bf348b538, 0x59f111f1b605d019, 0x923f82a4af194f9b, 0xab1c5ed5da6d8118,
0xd807aa98a3030242, 0x12835b0145706fbe, 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2,
0x72be5d74f27b896f, 0x80deb1fe3b1696b1, 0x9bdc06a725c71235, 0xc19bf174cf692694,
0xe49b69c19ef14ad2, 0xefbe4786384f25e3, 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65,
0x2de92c6f592b0275, 0x4a7484aa6ea6e483, 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5,
0x983e5152ee66dfab, 0xa831c66d2db43210, 0xb00327c898fb213f, 0xbf597fc7beef0ee4,
0xc6e00bf33da88fc2, 0xd5a79147930aa725, 0x06ca6351e003826f, 0x142929670a0e6e70,
0x27b70a8546d22ffc, 0x2e1b21385c26c926, 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df,
0x650a73548baf63de, 0x766a0abb3c77b2a8, 0x81c2c92e47edaee6, 0x92722c851482353b,
0xa2bfe8a14cf10364, 0xa81a664bbc423001, 0xc24b8b70d0f89791, 0xc76c51a30654be30,
0xd192e819d6ef5218, 0xd69906245565a910, 0xf40e35855771202a, 0x106aa07032bbd1b8,
0x19a4c116b8d2d0c8, 0x1e376c085141ab53, 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8,
0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb, 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3,
0x748f82ee5defb2fc, 0x78a5636f43172f60, 0x84c87814a1f0ab72, 0x8cc702081a6439ec,
0x90befffa23631e28, 0xa4506cebde82bde9, 0xbef9a3f7b2c67915, 0xc67178f2e372532b,
0xca273eceea26619c, 0xd186b8c721c0c207, 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178,
0x06f067aa72176fba, 0x0a637dc5a2c898a6, 0x113f9804bef90dae, 0x1b710b35131c471b,
0x28db77f523047d84, 0x32caab7b40c72493, 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c,
0x4cc5d4becb3e42b6, 0x597f299cfc657e2a, 0x5fcb6fab3ad6faec, 0x6c44198c4a475817]
def _rit_rot(on: int, by: int) -> int:
"""
helper function for right rotation as it isn't done by a simple bitwise operation (xor is done by '^')
:param on: value to be rotated
:param by: value by which to rotate
:return: right rotated 'on'
"""
return ((on >> by) | (on << (64 - by))) & 0xFFFFFFFFFFFFFFFF
def hash_main(chunk):
global _initial_hashes, _round_constants
# start the hashing process
# to begin, create a place to store the 80 words that we'll make
words = [0] * 80
# first 16 words will be saved without any changes
words[:16] = struct.unpack('!16Q', chunk)
# extend these 16 words into the remaining 64 words of 'message schedule array'
for i in range(16, 80):
part_1 = _rit_rot(words[i - 15], 1) ^ _rit_rot(words[i - 15], 8) ^ (words[i - 15] >> 7)
part_2 = _rit_rot(words[i - 2], 19) ^ _rit_rot(words[i - 2], 61) ^ (words[i - 2] >> 6)
words[i] = (words[i - 16] + part_1 + words[i - 7] + part_2) & 0xFFFFFFFFFFFFFFFF
# create the working variables
a, b, c, d, e, f, g, h = _initial_hashes
# start the compression function
for z in range(80):
var_1 = _rit_rot(a, 28) ^ _rit_rot(a, 34) ^ _rit_rot(a, 39)
var_2 = _rit_rot(e, 14) ^ _rit_rot(e, 18) ^ _rit_rot(e, 41)
var_3 = (a & b) ^ (a & c) ^ (b & c)
var_4 = (e & f) ^ ((~e) & g)
temp_1 = var_1 + var_3
temp_2 = h + var_2 + var_4 + _round_constants[z] + words[z]
# remix the hashes
h = g
g = f
f = e
e = (d + temp_2) & 0xFFFFFFFFFFFFFFFF
d = c
c = b
b = a
a = (temp_1 + temp_2) & 0xFFFFFFFFFFFFFFFF
# add this chunk to initial hashes
_initial_hashes = [(x + y) & 0xFFFFFFFFFFFFFFFF for x, y in zip(_initial_hashes,
[a, b, c, d, e, f, g, h])]
def _sha_backend_update(text_copy, _buffer, _counter):
"""
backend function that hashes given string
"""
global _initial_hashes, _round_constants
# create variables for cycling
_buffer += text_copy
_counter += len(text_copy)
# assert the variables are correct
if not text_copy:
return
if type(text_copy) is not str:
raise TypeError("Invalid Object! Please enter a valid string for hashing!")
# break the buffer into 128-bit chunks
while len(_buffer) >= 128:
chunk = _buffer[:128].encode()[1:]
hash_main(chunk)
_buffer = _buffer[128:]
def sha_backend_digest(text_to_hash: str, _buffer: str, _counter: int,
_output_size: int, hex_output: bool = False):
# initialize variables
variable_x = _counter & 0x7F
length = str(struct.pack('!Q', _counter << 3))
# set the thresholds
if variable_x < 112:
padding_len = 111 - variable_x
else:
padding_len = 239 - variable_x
# make a copy of the text_to_hash before starting hashing
text_copy = copy.deepcopy(text_to_hash)
m = '\x80' + ('\x00' * (padding_len + 8)) + length
# run the update function
_sha_backend_update(text_copy, _buffer, _counter)
# return the hash value
return_val = [hex(stuff) for stuff in _initial_hashes[:_output_size]]
if hex_output is True:
return_val = [int(stuff, base=16) for stuff in return_val]
return return_val
return ''.join(return_val)
def sha_512(text_to_hash: str, hex_digest: bool = False) -> str:
"""
frontend function for SHA512 hashing
:return: hashed string
"""
# before anything, check if the input is correct
if not text_to_hash:
return ""
if type(text_to_hash) is not str:
raise TypeError("Invalid content! Please provide content in correct format for hashing!")
# initialize default variables
_buffer = ''
_counter = 0
_output_size = 8
# start the backend function
return sha_backend_digest(text_to_hash, _buffer, _counter, _output_size, hex_output=hex_digest)
message = "This is a string to be hashed"
from hashlib import sha512
print("hashlib gives: ", sha512(message.encode()).hexdigest())
print("I give: ", sha_512(message))
As is obvious, I don't understand a lot of things in this algorithm and have literally copied many parts from the original code (also, I know it isn't good practice to write everything in a single function but I find it easier when trying to understand something).
But the biggest problem I have right now is it doesn't work! Whatever input message I provide to my function, it gives the same output:
0x6a09e667f3bcc9080xbb67ae8584caa73b0x3c6ef372fe94f82b0xa54ff53a5f1d36f1
0x510e527fade682d10x9b05688c2b3e6c1f0x1f83d9abfb41bd6b0x5be0cd19137e2179
I wrote a code at the bottom to compare it with python's hashlib module.
Where am I going wrong in this and how do I fix this?
EDIT: As mentioned in the comments, I tried to feed in a longer message string and the code seems to be working (it still gives longer output than hashlib though):
message = "This is a string to be hashed. I'll try to make this string as long as possible by adding" \
"as much information to it as I can, in the hopes that this string would somehow become longer than" \
"128 bits and my code can run properly. Hopefully, this is already longer than 128 bits, so lets see" \
"how it works..."
hash: 0x6fcc0f346f2577800x334bd9b6c1178a970x90964a3f45f7b5bb0xc14033d12f6607e60xb598bea0a8b0ac1e0x116b0e134691ab540x73d88e77e5b862ba0x89181da7462c5574
message = "This is a string to be hashed. I'll try to make this string as long as possible by adding" \
"as much information to it as I can, in the hopes that this string would somehow become longer than"
hash: 0x166e40ab03bc98750xe81fe34168b6994f0xe56b81bd5972b5560x8789265c3a56b30b0x2c810d652ea7b1550xa23ca2704602a8240x12ffb1ec8f3dd6d10x88c29f84cbef8988
You'll always have to pad the message. Padding and adding the length are always required as last step of the SHA-2 process. Currently you weren't performing that last step (to completion).
Here are my last two comments that pointed you in the right direction:
So generally you try and take one 128 byte block from the binary message, update the hash state using the information in that block, then move to the next one until you have a partial or 0 byte block. That block you need to pad & add size indication (in bits) and process. If you've not enough space for the padding / size indication then you need yet another block consisting entirely of padding and the size indication. If you read carefully, then you always process at least one block.
and
Hmm, it is already in sha_backend_digest (the 0x80 followed by zero bytes and the length which is input size * 8 (_counter << 3).
But of course you do need to perform that and not skip any step.
I have a file that I want to convert into custom base (base 86 for example, with custom alphabet)
I have try to convert the file with hexlify and then into my custom base but it's too slow... 8 second for 60 Ko..
def HexToBase(Hexa, AlphabetList, OccurList, threshold=10):
number = int(Hexa,16) #base 16 vers base 10
alphabet = GetAlphabet(AlphabetList, OccurList, threshold)
#GetAlphabet return a list of all chars that occurs more than threshold times
b_nbr = len(alphabet) #get the base
out = ''
while number > 0:
out = alphabet[(number % b_nbr)] + out
number = number // b_nbr
return out
file = open("File.jpg","rb")
binary_data = file.read()
HexToBase(binascii.hexlify(binary_data),['a','b'],[23,54])
So, could anyone help me to find the right solution ?
Sorry for my poor English I'm French, and Thank's for your help !
First you can replace:
int(binascii.hexlify(binary_data), 16) # timeit: 14.349809918712538
By:
int.from_bytes(binary_data, byteorder='little') # timeit: 3.3330371951720164
Second you can use the divmod function to speed up the loop:
out = ""
while number > 0:
number, m = divmod(number, b_nbr)
out = alphabet[m] + out
# timeit: 3.8345545611298126 vs 7.472579440019706
For divmod vs %, // comparison and large numbers, see Is divmod() faster than using the % and // operators?.
(Remark: I expected that buildind an array and then making a string with "".join would be faster than out = ... + out but that was not the case with CPython 3.6.)
Everything put together gave me a speed up factor of 6.
Problem:
I made an elf executable that self modifies one of its byte. It simply changes a 0 for a 1. When I run the executable normally, I can see that the change was successful because it runs exactly as expected (more on that further down). The problem arises when debugging it: The debugger (using radare2) returns the wrong value when looking at the modified byte.
Context:
I made a reverse engineering challenge, inspired by Smallest elf. You can see the "source code" (if you can even call it that) there: https://pastebin.com/Yr1nFX8W.
To assemble and execute:
nasm -f bin -o tinyelf tinyelf.asm
chmod +x tinyelf
./tinyelf [flag]
If the flag is right, it returns 0. Any other value means your answer is wrong.
./tinyelf FLAG{wrong-flag}; echo $?
... outputs "255".
!Solution SPOILERS!
It's possible to reverse it statically. Once that is done, you will find out that each characters in the flag is found by doing this calculation:
flag[i] = b[i] + b[i+32] + b[i+64] + b[i+96];
...where i is the index of the character, and b is the bytes of the executable itself. Here is a c script that solve the challenge without a debugger:
#include <stdio.h>
int main()
{
char buffer[128];
FILE* fp;
fp = fopen("tinyelf", "r");
fread(buffer, 128, 1, fp);
int i;
char c = 0;
for (i = 0; i < 32; i++) {
c = buffer[i];
// handle self-modifying code
if (i == 10) {
c = 0;
}
c += buffer[i+32] + buffer[i+64] + buffer[i+96];
printf("%c", c);
}
printf("\n");
}
You can see that my solver handles a special case: When i == 10, c = 0. That's because it's the index of the byte that is modified during execution. Running the solver and calling tinyelf with it I get:
FLAG{Wh3n0ptiMizaTioNGOesT00F4r}
./tinyelf FLAG{Wh3n0ptiMizaTioNGOesT00F4r} ; echo $?
Output: 0. Success!
Great, let's try to solve it dynamically now, using python and radare2:
import r2pipe
r2 = r2pipe.open('./tinyelf')
r2.cmd('doo FLAG{AAAAAAAAAAAAAAAAAAAAAAAAAA}')
r2.cmd('db 0x01002051')
flag = ''
for i in range(0, 32):
r2.cmd('dc')
eax = r2.cmd('dr? al')
c = int(eax, 16)
flag += chr(c)
print('\n\n' + flag)
It puts a breakpoint on the command that compares the input characters with the expected characters, then get what it is compared with (al). This SHOULD work. Yet, here is the output:
FLAG{Wh3n0�tiMiza�ioNGOesT00F4r}
2 incorrect values, one of which is at the index 10 (the modified byte). Weird, maybe a bug with radare2? Let's try unicorn (a cpu emulator) next:
from unicorn import *
from unicorn.x86_const import *
from pwn import *
ADDRESS = 0x01002000
mu = Uc(UC_ARCH_X86, UC_MODE_32)
code = bytearray(open('./tinyelf').read())
mu.mem_map(ADDRESS, 20 * 1024 * 1024)
mu.mem_write(ADDRESS, str(code))
mu.reg_write(UC_X86_REG_ESP, ADDRESS + 0x2000)
mu.reg_write(UC_X86_REG_EBP, ADDRESS + 0x2000)
mu.mem_write(ADDRESS + 0x2000, p32(2)) # argc
mu.mem_write(ADDRESS + 0x2000 + 4, p32(ADDRESS + 0x5000)) # argv[0]
mu.mem_write(ADDRESS + 0x2000 + 8, p32(ADDRESS + 0x5000)) # argv[1]
mu.mem_write(ADDRESS + 0x5000, "x" * 32)
flag = ''
def hook_code(uc, address, size, user_data):
global flag
eip = uc.reg_read(UC_X86_REG_EIP)
if eip == 0x01002051:
c = uc.reg_read(UC_X86_REG_EAX) & 0x7f
#print(str(c) + " " + chr(c))
flag += chr(c)
mu.hook_add(UC_HOOK_CODE, hook_code)
try:
mu.emu_start(0x01002004, ADDRESS + len(code))
except Exception:
print flag
This time the solver outputs: FLAG{Wh3n0otiMizaTioNGOesT00F4r}
Notice at the index 10: 'o' instead of 'p'. That's an off by 1 mistake exactly where the byte is modified. That can't be a coincidence, right?
Anyone has an idea why both these scripts do not work? Thank you.
There is no issue with radare2 but your analysis of the program is incorrect thus the code that you wrote handles this RE incorrectly.
Lets start with
When i == 10, c = 0. That's because it's the index of the byte that is modified during execution.
That is partially true. It is set to zero at the beginning but then after each round there is this code:
xor al, byte [esi]
or byte [ebx + 0xa], al
So let's understand what's happening here. al is the currently calculated char of the flag and esi points to the FLAG that was entered as a argument and at [ebx + 0xa] we currently have 0 (set at the beginning), so the char at index 0xa will stay zero only if the calculated flag char is equal to the one in argument and since you are running r2 with a fake flag, that starts to be a problem from 6th char but the result of this you see at the first � at index 10. To mitigate that we need to update your script a little bit.
eax = r2.cmd('dr? al')
c = int(eax, 16)
r2.cmd("ds 2")
r2.cmd("dr al = 0x0")
What we do here is that after the brekpoint was hit and we read the calculated flag char we move two instructions further (to reach 0x01002054) and then we set al to 0x0 to emulate that our char at [esi] was actually the same as the calculated one (so xor will return 0 in such case). By doing this we keep value at 0xa to be zero still.
Now the second character. This RE is tricky ;) - it reads itself and if you forget about that you might end up with case like this. Let's try to analyze why this character is off. It is 18th character of the flag (so index is 17 as we start from 0) and if we check the formula for characters indexes that we read from the binary we noticed that indexes are: 17(dec) = 11(hex), 17 + 32 = 49(dec) = 31(hex), 17 + 64 = 81(dec) = 51(hex), 17 + 96 = 113(dec) = 71(hex). But this 51(hex) looks oddly familiar? Didn't we see that somewhere before? Yup, it's the offset at which you set your breakpoint to read the al value.
This is the code that break your second char
r2.cmd('db 0x01002051')
Yup - your breakpoint. You are setting to break at that address and a soft breakpoint is putting a 0xcc in the memory address so when the opcode that reads 3rd byte of the 18th char hits that spot it does not get 0x5b (the original value) it gets 0xcc. So to fix that we need to correct that calculation. Here probably it can be done in a smarter/more elegant way but I went for a simple solution so I just did this:
if i == 17:
c -= (0xcc-0x5b)
Just subtract was was unintentionally added by putting a breakpoint in the code.
The final code:
import r2pipe
r2 = r2pipe.open('./tinyelf')
print r2
r2.cmd("doo FLAG{AAAAAAAAAAAAAAAAAAAAAAAAAA}")
r2.cmd("db 0x01002051")
flag = ''
for i in range(0, 32):
r2.cmd("dc")
eax = r2.cmd('dr? al')
c = int(eax, 16)
if i == 17:
c -= (0xcc-0x5b)
r2.cmd("ds 2")
r2.cmd("dr al = 0x0")
flag += chr(c)
print('\n\n' + flag)
That prints the correct flag:
FLAG{Wh3n0ptiMizaTioNGOesT00F4r}
As for the Unicorn you are not setting the breakpoint so the problem 2 goes away, and the off-by-1 on 10th index is due to the same reason as for r2.
I want to get the length of a string including a part of the string that represents its own length without padding or using structs or anything like that that forces fixed lengths.
So for example I want to be able to take this string as input:
"A string|"
And return this:
"A string|11"
On the basis of the OP tolerating such an approach (and to provide an implementation technique for the eventual python answer), here's a solution in Java.
final String s = "A String|";
int n = s.length(); // `length()` returns the length of the string.
String t; // the result
do {
t = s + n; // append the stringified n to the original string
if (n == t.length()){
return t; // string length no longer changing; we're good.
}
n = t.length(); // n must hold the total length
} while (true); // round again
The problem of, course, is that in appending n, the string length changes. But luckily, the length only ever increases or stays the same. So it will converge very quickly: due to the logarithmic nature of the length of n. In this particular case, the attempted values of n are 9, 10, and 11. And that's a pernicious case.
A simple solution is :
def addlength(string):
n1=len(string)
n2=len(str(n1))+n1
n2 += len(str(n2))-len(str(n1)) # a carry can arise
return string+str(n2)
Since a possible carry will increase the length by at most one unit.
Examples :
In [2]: addlength('a'*8)
Out[2]: 'aaaaaaaa9'
In [3]: addlength('a'*9)
Out[3]: 'aaaaaaaaa11'
In [4]: addlength('a'*99)
Out[4]: 'aaaaa...aaa102'
In [5]: addlength('a'*999)
Out[5]: 'aaaa...aaa1003'
Here is a simple python port of Bathsheba's answer :
def str_len(s):
n = len(s)
t = ''
while True:
t = s + str(n)
if n == len(t):
return t
n = len(t)
This is a much more clever and simple way than anything I was thinking of trying!
Suppose you had s = 'abcdefgh|, On the first pass through, t = 'abcdefgh|9
Since n != len(t) ( which is now 10 ) it goes through again : t = 'abcdefgh|' + str(n) and str(n)='10' so you have abcdefgh|10 which is still not quite right! Now n=len(t) which is finally n=11 you get it right then. Pretty clever solution!
It is a tricky one, but I think I've figured it out.
Done in a hurry in Python 2.7, please fully test - this should handle strings up to 998 characters:
import sys
orig = sys.argv[1]
origLen = len(orig)
if (origLen >= 98):
extra = str(origLen + 3)
elif (origLen >= 8):
extra = str(origLen + 2)
else:
extra = str(origLen + 1)
final = orig + extra
print final
Results of very brief testing
C:\Users\PH\Desktop>python test.py "tiny|"
tiny|6
C:\Users\PH\Desktop>python test.py "myString|"
myString|11
C:\Users\PH\Desktop>python test.py "myStringWith98Characters.........................................................................|"
myStringWith98Characters.........................................................................|101
Just find the length of the string. Then iterate through each value of the number of digits the length of the resulting string can possibly have. While iterating, check if the sum of the number of digits to be appended and the initial string length is equal to the length of the resulting string.
def get_length(s):
s = s + "|"
result = ""
len_s = len(s)
i = 1
while True:
candidate = len_s + i
if len(str(candidate)) == i:
result = s + str(len_s + i)
break
i += 1
This code gives the result.
I used a few var, but at the end it shows the output you want:
def len_s(s):
s = s + '|'
b = len(s)
z = s + str(b)
length = len(z)
new_s = s + str(length)
new_len = len(new_s)
return s + str(new_len)
s = "A string"
print len_s(s)
Here's a direct equation for this (so it's not necessary to construct the string). If s is the string, then the length of the string including the length of the appended length will be:
L1 = len(s) + 1 + int(log10(len(s) + 1 + int(log10(len(s)))))
The idea here is that a direct calculation is only problematic when the appended length will push the length past a power of ten; that is, at 9, 98, 99, 997, 998, 999, 9996, etc. To work this through, 1 + int(log10(len(s))) is the number of digits in the length of s. If we add that to len(s), then 9->10, 98->100, 99->101, etc, but still 8->9, 97->99, etc, so we can push past the power of ten exactly as needed. That is, adding this produces a number with the correct number of digits after the addition. Then do the log again to find the length of that number and that's the answer.
To test this:
from math import log10
def find_length(s):
L1 = len(s) + 1 + int(log10(len(s) + 1 + int(log10(len(s)))))
return L1
# test, just looking at lengths around 10**n
for i in range(9):
for j in range(30):
L = abs(10**i - j + 10) + 1
s = "a"*L
x0 = find_length(s)
new0 = s+`x0`
if len(new0)!=x0:
print "error", len(s), x0, log10(len(s)), log10(x0)
How do I convert a hex string to a signed int in Python 3?
The best I can come up with is
h = '9DA92DAB'
b = bytes(h, 'utf-8')
ba = binascii.a2b_hex(b)
print(int.from_bytes(ba, byteorder='big', signed=True))
Is there a simpler way? Unsigned is so much easier: int(h, 16)
BTW, the origin of the question is itunes persistent id - music library xml version and iTunes hex version
In n-bit two's complement, bits have value:
bit 0 = 20
bit 1 = 21
bit n-2 = 2n-2
bit n-1 = -2n-1
But bit n-1 has value 2n-1 when unsigned, so the number is 2n too high. Subtract 2n if bit n-1 is set:
def twos_complement(hexstr, bits):
value = int(hexstr, 16)
if value & (1 << (bits - 1)):
value -= 1 << bits
return value
print(twos_complement('FFFE', 16))
print(twos_complement('7FFF', 16))
print(twos_complement('7F', 8))
print(twos_complement('FF', 8))
Output:
-2
32767
127
-1
import struct
For Python 3 (with comments' help):
h = '9DA92DAB'
struct.unpack('>i', bytes.fromhex(h))
For Python 2:
h = '9DA92DAB'
struct.unpack('>i', h.decode('hex'))
or if it is little endian:
h = '9DA92DAB'
struct.unpack('<i', h.decode('hex'))
Here's a general function you can use for hex of any size:
import math
# hex string to signed integer
def htosi(val):
uintval = int(val,16)
bits = 4 * (len(val) - 2)
if uintval >= math.pow(2,bits-1):
uintval = int(0 - (math.pow(2,bits) - uintval))
return uintval
And to use it:
h = str(hex(-5))
h2 = str(hex(-13589))
x = htosi(h)
x2 = htosi(h2)
This works for 16 bit signed ints, you can extend for 32 bit ints. It uses the basic definition of 2's complement signed numbers. Also note xor with 1 is the same as a binary negate.
# convert to unsigned
x = int('ffbf', 16) # example (-65)
# check sign bit
if (x & 0x8000) == 0x8000:
# if set, invert and add one to get the negative value, then add the negative sign
x = -( (x ^ 0xffff) + 1)
It's a very late answer, but here's a function to do the above. This will extend for whatever length you provide. Credit for portions of this to another SO answer (I lost the link, so please provide it if you find it).
def hex_to_signed(source):
"""Convert a string hex value to a signed hexidecimal value.
This assumes that source is the proper length, and the sign bit
is the first bit in the first byte of the correct length.
hex_to_signed("F") should return -1.
hex_to_signed("0F") should return 15.
"""
if not isinstance(source, str):
raise ValueError("string type required")
if 0 == len(source):
raise valueError("string is empty")
sign_bit_mask = 1 << (len(source)*4-1)
other_bits_mask = sign_bit_mask - 1
value = int(source, 16)
return -(value & sign_bit_mask) | (value & other_bits_mask)