We Keep Coding

nodejs crypto-js vs python sha256

I have a problem with crypto-js vs python sha256. It would like to write a nodejs client for duplicati. So I tried to port some python code to js.
https://github.com/Pectojin/duplicati-client/blob/master/auth.py#L136
Javascript
const CryptoJS = require('crypto-js');
function sha256(to_sign) {
var hash = CryptoJS.SHA256(to_sign.toString());
var hashInBase64 = CryptoJS.enc.Base64.stringify(hash);
return hashInBase64.toString('utf-8');
}
let salt = "ZAwQqEAAwR78oZOxFu0nVH2FLy/BnulVxhuu9IOnBwg="
let salt2 = "YQ=="
let password = "abc"
let saltedpwd = sha256(Buffer.concat([Buffer.from(password),Buffer.from(salt,'base64')]));
let saltedpwd2 = sha256(Buffer.concat([Buffer.from(password),Buffer.from(salt2,'base64')]));
let new_password = saltedpwd.toString('base64');
let new_password2 = saltedpwd2.toString('base64');
console.log(new_password)
console.log(new_password2)
returns:
pw1: 0udYFffMXd2QWW9dVXbFl3qp/6lnRcnspr4M1VEtgJA=
pw2: XD9nt/qE374RGDh8rRR5OSmEWlvHwAgMTYMJ03uqaNA=
Python
import base64
import hashlib
import sys
password = "abc"
salt = "ZAwQqEAAwR78oZOxFu0nVH2FLy/BnulVxhuu9IOnBwg="
salt2 = "YQ=="
salt_password = password.encode() + base64.b64decode(salt)
saltedpwd = hashlib.sha256(salt_password).digest()
print (base64.b64encode(saltedpwd).decode('utf-8'))
salt_password2 = password.encode() + base64.b64decode(salt2)
saltedpwd2 = hashlib.sha256(salt_password2).digest()
print (base64.b64encode(saltedpwd2).decode('utf-8'))
returns:
pw1: v9bAzxPatGzA2W7ORkraUvh+nyXotWXItAKpawGSo+A=
pw2: XD9nt/qE374RGDh8rRR5OSmEWlvHwAgMTYMJ03uqaNA=
as you can see, pw2 with the very simple base64 salt is both the same.
the salt from pw1 comes from the duplicati server, so I can't control it...
I've tried some many combinations of encoding, CryptoJS options, so I'm at the point where I will soon stop my 'project'... :(
Can you please give me any advice, what I'm doing wrong? I would be glad about any information.
Regards,
Benjamin

You're 99% of the way there, I think the fix is a one liner (isn't it so often!).
We just have to change
var hash = CryptoJS.SHA256(to_sign.toString());
to
var hash = CryptoJS.SHA256(CryptoJS.lib.WordArray.create(to_sign));
I believe this is because we want to convert directly from the buffer we created by concatenating the password and salt, rather than converting to a string, which is causing the wrong hash to be computed.
In any case, we get the same output as the Python code, which is what we want, ie
v9bAzxPatGzA2W7ORkraUvh+nyXotWXItAKpawGSo+A=
XD9nt/qE374RGDh8rRR5OSmEWlvHwAgMTYMJ03uqaNA=
The new code would look like this in Node.js:
const CryptoJS = require('crypto-js');
function sha256(to_sign) {
var hash = CryptoJS.SHA256(CryptoJS.lib.WordArray.create(to_sign));
var hashInBase64 = CryptoJS.enc.Base64.stringify(hash);
return hashInBase64.toString('utf-8');
}
let salt = "ZAwQqEAAwR78oZOxFu0nVH2FLy/BnulVxhuu9IOnBwg="
let salt2 = "YQ=="
let password = "abc"
let saltedpwd = sha256(Buffer.concat([Buffer.from(password),Buffer.from(salt,'base64')]));
let saltedpwd2 = sha256(Buffer.concat([Buffer.from(password),Buffer.from(salt2,'base64')]));
let new_password = saltedpwd.toString('base64');
let new_password2 = saltedpwd2.toString('base64');
console.log(new_password)
console.log(new_password2)

Convert from Tensorflow -> CoreML 3.0 for slot/intent detection

I am trying to use some of the models created by this codebase (Slot-Filling-Understanding-Using-RNNs) in my Swift application.
I was able to convert lstm_nopooling, lstm_nopooling300 and lstm to convert to CoreML.
In model.py I used this code:
def save_model(self):
joblib.dump(self.summary, 'models/' + self.name + '.txt')
self.model.save('models/' + self.name + '.h5')
try:
coreml_model = coremltools.converters.keras.convert(self.model, input_names="main_input", output_names=["intent_output","slot_output"])
coreml_model.save('models/' + self.name + '.mlmodel')
except:
pass
print("Saved model to disk")
I am trying to convert the vectors back to an intent and slots.
I have this, but
func tokenizeSentences(instr: String) -> [Int] {
let s = instr.lowercased().split(separator: " ")
var ret = [Int]()
if let filepath = Bundle.main.path(forResource: "atis.dict.vocab", ofType: "csv") {
do {
let contents = try String(contentsOfFile: filepath)
print(contents)
var lines = contents.split { $0.isNewline }
var pos = 0
for word in s {
if let index = lines.firstIndex(of: word) {
print(index.description + " " + word)
ret.append(index)
}
}
return ret
} catch {
// contents could not be loaded
}
} else {
// example.txt not found!
}
return ret
}
func predictText(instr:String) {
let model = lstm_nopooling300()
guard let mlMultiArray = try? MLMultiArray(shape:[20,1,1],
dataType:MLMultiArrayDataType.int32) else {
fatalError("Unexpected runtime error. MLMultiArray")
}
let tokens = tokenizeSentences(instr: instr)
for (index, element) in tokens.enumerated() {
mlMultiArray[index] = NSNumber(integerLiteral: element)
}
guard let m = try? model.prediction(input: lstm_nopooling300Input.init(main_input: mlMultiArray))
else {
fatalError("Unexpected runtime error. MLMultiArray")
}
let mm = m.intent_output
let length = mm.count
let doublePtr = mm.dataPointer.bindMemory(to: Double.self, capacity: length)
let doubleBuffer = UnsafeBufferPointer(start: doublePtr, count: length)
let output = Array(doubleBuffer)
print("******** intents \(mm.count) ********")
print(output)
let mn = m.slot_output
let length2 = mn.count
let doublePtr2 = mm.dataPointer.bindMemory(to: Double.self, capacity: length2)
let doubleBuffer2 = UnsafeBufferPointer(start: doublePtr2, count: length2)
let output2 = Array(doubleBuffer2)
print("******** slots \(mn.count) ********")
print(output2)
}
}
When I run my code I get this, truncated, for intents:
******** intents 540 ********
[0.0028914143331348896, 0.0057610333897173405, 4.1651015635579824e-05,
0.15935245156288147, 5.6665314332349226e-05, 5.7797817134996876e-05, 0.0044302307069301605, 0.00012486864579841495, 0.0004683282459154725, 0.003053907072171569, 3.806956738117151e-05, 0.012112349271774292, 5.861848694621585e-05, 0.0031344725284725428,
The problem, I believe, is that the ids are in a pickle file, so in atis/atis.train.pkl perhaps.
All I did was train the models and convert those I could to CoreML and now I am trying to use it, but not certain what to do next.
I have a textfield and I enter 'current weather in london' and I hope to get something similar to (this is from running example.py)
{'intent': 'weather_intent', 'slots': [{'name': 'city', 'value': 'London'}]}
Here is the coreml input/output

Thanks to #MatthijsHollemans I was able to figure out what to do.
In data_processing.py I added these:
with open('atis/wordlist.csv', 'w') as f:
for key in ids2words.keys():
f.write("%s\n"%(ids2words.keys[key]))
with open('atis/wordlist_slots.csv', 'w') as f:
for key in ids2slots.keys():
f.write("%s\n"%(ids2slots[key]))
with open('atis/wordlist_intents.csv', 'w') as f:
for key in ids2intents.keys():
f.write("%s\n"%(ids2intents[key]))
This allows me to tokenize correctly, using wordlist.csv.
Then when I get the response back, and using mm.count was wrong, it should have been output.count for example, I could see the intents.
Look for the element that has the largest value, and then you look up in wordlist_intents.csv (I turned this into an array, probably should be a dictionary) to find the likely intent.
I still need to do the slots, but the basic idea is the same.
The key was to output the dictionary used in python to a csv file and then import that into the project.
UPDATE
I realized that when mm.count was 540 that is because it can have 20 words in the sentence and so it can return that many. So in my case I needed to split the word by space and then loop that many times as I won't get more slots than I have words.
I am doing this in SwiftUI so I also had to create an observable so I could use EnvironmentObject to pass in the terms.
So, to properly loop over the double array in memory I am including the latest code that does what I expect.
func predictText(instr:String) {
let model = lstm_nopooling300()
guard let mlMultiArray = try? MLMultiArray(shape:[20,1,1],
dataType:MLMultiArrayDataType.int32) else {
fatalError("Unexpected runtime error. MLMultiArray")
}
let tokens = tokenizeSentences(instr: instr)
let sent = instr.split(separator: " ")
print(instr)
print(tokens)
for (index, element) in tokens.enumerated() {
mlMultiArray[index] = NSNumber(integerLiteral: element)
}
guard let m = try? model.prediction(input: lstm_nopooling300Input.init(main_input: mlMultiArray))
else {
fatalError("Unexpected runtime error. MLMultiArray")
}
let mm = m.intent_output
let length = mm.count
let doublePtr = mm.dataPointer.bindMemory(to: Double.self, capacity: length)
var intents = [String]()
for i in 0...sent.count - 1 {
let doubleBuffer = UnsafeBufferPointer(start: doublePtr + i * 27, count: 27)
let output = Array(doubleBuffer)
let intent = convertVectorToIntent(vector: output)
intents.append(intent)
}
print(intents)
let mn = m.slot_output
let length2 = mn.count
let doublePtr2 = mn.dataPointer.bindMemory(to: Double.self, capacity: length2)
var slots = [String]()
for i in 0...sent.count - 1 {
let doubleBuffer2 = UnsafeBufferPointer(start: doublePtr2 + i * 133, count: 133)
let output2 = Array(doubleBuffer2)
var slot = ""
slot = convertVectorToSlot(vector: output2)
slots.append(slot)
slots.append(sent[i].description)
}
print(slots)
}

Hashing algorithm Node js vs Python

I was trying to convert a hash algorithm which is written on Python to node.js
The python code looks something as
import uuid
import hashlib
import struct
CLIENT_ID = uuid.UUID('c5f92e0d-e762-32cd-98cb-8c546c410dbe')
SECRET = uuid.UUID('2cf26ff5-bd06-3245-becf-4d5a3baa704f')
data = CLIENT_ID.bytes_le + SECRET.bytes_le + struct.pack("I", 2017) + struct.pack("I", 9) + struct.pack("I", 2)
token = str(uuid.UUID(bytes_le=hashlib.sha256(data).digest()[0:16]))
The token generated is 32d86f00-eb49-2739-e957-91513d2b9969
Here the date values struct.pack values are generated using datetime but for convenient I have hard coded here.
I tried to convert the same by looking at the python doc for the respective libraries and did so far as
let CLIENT_ID = new Buffer('c5f92e0d-e762-32cd-98cb-8c546c410dbe');
let SECRET = new Buffer('2cf26ff5-bd06-3245-becf-4d5a3baa704f');
let d = new Buffer(2);
let m = new Buffer(9);
let y = new Buffer(2017);
let data = CLIENT_ID+SECRET+y+m+d;
const uuidv4 = require('uuid/v4');
const hash = crypto.createHash('sha256');
let token = uuidv4({random: hash.update(data, 'utf8').digest().slice(0, 16)}, 0);
And the hash it generates is b7b82474-eab4-4295-8318-cc258577ff9b
So, basically I am miserably missing something for the nodejs part.
Could you please guide me on where what went wrong. Thanks for the help

There's a lot of missed parts actually it tuned out.
###node parts:
new Buffer('c5')
does not represent <Buffer c5>, but <Buffer 63 35>.
To write c5 you would need to use Buffer.from([0xc5]) or Buffer.from([197]) (dec).
new Buffer(2)
does not represent <Buffer 02>, it just allocates 2 bytes.
CLIENT_ID+SECRET+y+m+d
concatenation of buffers does not work that way.
Use array of buffers and Buffer.concat([buffers]) to concatenate buffers.
###uuid parts:
it turned out that uuid operates modified version of buffers (bytes_le part in python code)
#the most interesting part:
in the python version of uuid, if no version argument is passed to uuid.UUID(...), uuid would generate an ID without fixing bits
According to the RFC-4122 4.4 uuid should fix that bits.
uuid.py skips RFC-4122 4.4
node-uuid/v4.js fixes required bits
that way even with the same results for sha256 hashing, the results between python and node implementation still would differ
python: 32d86f00-eb49-2739-e957-91513d2b9969
node: 32d86f00-eb49-4739-a957-91513d2b9969
^ ^
So, I see here 2 options
to pass version to python uuid (only for the last uuid call uuid.UUID(bytes_le=..., version=4)), that way python would return 32d86f00-eb49-4739-a957-91513d2b9969
if there's no way to change source code in python project, I guess there's an option to fork uuid and remove two lines of code in node-uuid/v4.js?
##See node version of your code below:
const uuidv4 = require('uuid/v4');
const crypto = require('crypto');
const hash = crypto.createHash('sha256');
const client_id_hex_str = "c5f92e0d-e762-32cd-98cb-8c546c410dbe".replace(/-/g, "");
const secret_hex_str = "2cf26ff5-bd06-3245-becf-4d5a3baa704f".replace(/-/g, "");
let CLIENT_ID = Buffer.from(to_bytes_le(to_bytes(client_id_hex_str, null, 16, 'big')));
let SECRET = Buffer.from(to_bytes_le(to_bytes(secret_hex_str, null, 16, 'big')));
let d = Buffer.from(to_bytes(null, 2, 4));
let m = Buffer.from(to_bytes(null, 9, 4));
let y = Buffer.from(to_bytes(null, 2017, 4));
let data = Buffer.concat([CLIENT_ID, SECRET, y, m, d]);
let hashBytes = hash.update(data, 'utf8').digest().slice(0, 16);
hashBytes = [].slice.call(hashBytes, 0);
hashBytes = Buffer.from(to_bytes_le(hashBytes));
let token = uuidv4({random: hashBytes});
console.log(token);
// https://stackoverflow.com/questions/16022556/has-python-3-to-bytes-been-back-ported-to-python-2-7
function to_bytes(hexString, number, length, endianess) {
if (hexString == null && number == null) {
throw new Error("Missing hex string or number.");
}
if (!length || isNaN(length)) {
throw new Error("Missing or invalid bytes array length number.");
}
if (hexString && typeof hexString != "string") {
throw new Error("Invalid format for hex value.");
}
if (hexString == null) {
if (isNaN(number)) {
throw new Error("Invalid number.");
}
hexString = number.toString(16);
}
let byteArray = [];
if (hexString.length % 2 !== 0) {
hexString = '0' + hexString;
}
const bitsLength = length * 2
hexString = ("0".repeat(bitsLength) + hexString).slice(-1 * bitsLength);
for (let i = 0; i < hexString.length; i += 2) {
const byte = hexString[i] + hexString[i + 1];
byteArray.push(parseInt(byte, 16));
}
if (endianess !== "big") {
byteArray = byteArray.reverse();
}
return byteArray;
}
// https://github.com/python/cpython/blob/master/Lib/uuid.py#L258
function to_bytes_le(bytes) {
const p1 = bytes.slice(0, 4).reverse();
const p2 = bytes.slice(4, 6).reverse();
const p3 = bytes.slice(6, 8).reverse();
const p4 = bytes.slice(8);
const bytes_le = [].concat.apply([], [p1, p2, p3, p4]);
return bytes_le;
}

Do you want the hashing of the data to be the same as the Python code above?
If not, you can take a look at the the Sha256 module below in NodeJS
https://www.npmjs.com/package/sha256

How to access wmi in python?

So I am trying to access the data from here
in Python. As you can see, it uses wmi. I have tried to use wmi in python before but I am having trouble interpreting the data they are giving me. Please be patient with me as I am a noob to how wmi works. It says that the wmi data is stored in root/OpenHardwareMontor and that it uses two different wmi classes(Hardware and Sensor). But all this information is going over my head.
could someone please give me some sample code to read some data from this?
For example, the code to check cpu core 1 frequency.
EDIT: i have sort of got it working. i run this code:
for Temperature in c.sensor():
print Temperature.identifier
print Temperature.value
and i get this:
/hdd/0/load/0
37.6608924866
/intelcpu/0/temperature/1
53.0
/intelcpu/0/temperature/0
42.0
/ram/data/1
2.88324356079
/intelcpu/0/load/2
1.53846144676
/hdd/0/temperature/0
43.0
/intelcpu/0/load/0
2.30768918991
/intelcpu/0/clock/1
1463.29663086
/intelcpu/0/clock/0
133.02696228
/intelcpu/0/clock/2
1463.29663086
/ram/load/0
49.224521637
/ram/data/0
2.79517364502
/intelcpu/0/load/1
3.07692289352
how can i request only the value associated with the identifier /intelcpu/0/temperature/1 ignoring all other values?

The most simple example to use WMI:
c = wmi.WMI()
wql = "Select * From Win32_SerialPort"
for item in c.query(wql):
print item
Output Example:
instance of Win32_SerialPort
{
Availability = 2;
Binary = TRUE;
Caption = "SpectrumAnalyzer1 (COM15)";
ConfigManagerErrorCode = 0;
ConfigManagerUserConfig = FALSE;
CreationClassName = "Win32_SerialPort";
Description = "SpectrumAnalyzer1";
DeviceID = "COM15";
MaxBaudRate = 128000;
MaximumInputBufferSize = 0;
MaximumOutputBufferSize = 0;
Name = "SpectrumAnalyzer1 (COM15)";
OSAutoDiscovered = TRUE;
PNPDeviceID = "USB\\VID_10C4&PID_ED00\\1269376";
PowerManagementCapabilities = {1};
PowerManagementSupported = FALSE;
ProviderType = "RS232 Serial Port";
SettableBaudRate = TRUE;
SettableDataBits = TRUE;
SettableFlowControl = TRUE;
SettableParity = TRUE;
SettableParityCheck = TRUE;
SettableRLSD = TRUE;
SettableStopBits = TRUE;
Status = "OK";
StatusInfo = 3;
Supports16BitMode = FALSE;
SupportsDTRDSR = TRUE;
SupportsElapsedTimeouts = TRUE;
SupportsIntTimeouts = TRUE;
SupportsParityCheck = TRUE;
SupportsRLSD = TRUE;
SupportsRTSCTS = TRUE;
SupportsSpecialCharacters = TRUE;
SupportsXOnXOff = TRUE;
SupportsXOnXOffSet = TRUE;
SystemCreationClassName = "Win32_ComputerSystem";
SystemName = ".......";
};
You can access each item by:
myQuery = c.query(wql)
myQuery.Availability
Output:
2
For more information, try the WMI cookbook.
Edit #1:
Using if statements and in you can do what you want.
for Temperature in c.sensor():
if "/intelcpu/0/temperature/1" in Temperature.identifier:
print Temperature.identifier
print Temperature.value

Retrieve User Entry IDs from MAPI

I extended the win32comext MAPI with the Interface IExchangeModifyTable to edit ACLs via the MAPI. I can modify existing ACL entries, but I stuck in adding new entries. I need the users entry ID to add it, according this C example
(Example Source from MSDN)
STDMETHODIMP AddUserPermission(
LPSTR szUserAlias,
LPMAPISESSION lpSession,
LPEXCHANGEMODIFYTABLE lpExchModTbl,
ACLRIGHTS frights)
{
HRESULT hr = S_OK;
LPADRBOOK lpAdrBook;
ULONG cbEid;
LPENTRYID lpEid = NULL;
SPropValue prop[2] = {0};
ROWLIST rowList = {0};
char szExName[MAX_PATH];
// Replace with "/o=OrganizationName/ou=SiteName/cn=Recipients/cn="
char* szServerDN = "/o=org/ou=site/cn=Recipients/cn=";
strcpy(szExName, szServerDN);
strcat(szExName, szUserAlias);
// Open the address book.
hr = lpSession->OpenAddressBook(0,
0,
MAPI_ACCESS_MODIFY,
&lpAdrBook );
if ( FAILED( hr ) ) goto cleanup;
// Obtain the entry ID for the recipient.
hr = HrCreateDirEntryIdEx(lpAdrBook,
szExName,
&cbEid,
&lpEid);
if ( FAILED( hr ) ) goto cleanup;
prop[0].ulPropTag = PR_MEMBER_ENTRYID;
prop[0].Value.bin.cb = cbEid;
prop[0].Value.bin.lpb = (BYTE*)lpEid;
prop[1].ulPropTag = PR_MEMBER_RIGHTS;
prop[1].Value.l = frights;
rowList.cEntries = 1;
rowList.aEntries->ulRowFlags = ROW_ADD;
rowList.aEntries->cValues = 2;
rowList.aEntries->rgPropVals = &prop[0];
hr = lpExchModTbl->ModifyTable(0, &rowList);
if(FAILED(hr)) goto cleanup;
printf("Added user permission. \n");
cleanup:
if (lpAdrBook)
lpAdrBook->Release();
return hr;
}
I can open the Address Book, but HrCreateDirEntryIdEx is not provided in the pywin32 mapi. I found it in the exchange extension, which does not compile on my system, the missing library problem. Do you have any idea to retrieve the users entry ID?
Thank.
Patrick

I got this piece of code and it works fine
from binascii import b2a_hex, a2b_hex
import active_directory as ad
# entry_type, see http://msdn.microsoft.com/en-us/library/cc840018.aspx
# + AB_DT_CONTAINER 0x000000100
# + AB_DT_TEMPLATE 0x000000101
# + AB_DT_OOUSER 0x000000102
# + AB_DT_SEARCH 0x000000200
# ab_flags, maybe see here: https://svn.openchange.org/openchange/trunk/libmapi/mapidefs.h
def gen_exchange_entry_id(user_id, ab_flags=0, entry_type = 0):
muidEMSAB = "DCA740C8C042101AB4B908002B2FE182"
version = 1
# Find user and bail out if it's not there
ad_obj = ad.find_user(user_id)
if not ad_obj:
return None
return "%08X%s%08X%08X%s00" % (
ab_flags,
muidEMSAB,
version,
entry_type,
b2a_hex(ad_obj.legacyExchangeDN.upper()).upper(),
)
data = gen_exchange_entry_id("myusername")
print data
print len(a2b_hex(data))