I'm trying to post events to Google Analytics. It works fine when I do it using the NodeJS code below, but fails when I use the Python code below. Both do return a HTTP 200 and even when posting to the debug URL (https://www.google-analytics.com/debug/collect) Google Analytics returns success details in both cases (see valid: true in the response below). The problem is that when posting from NodeJS the result shows up in the GA website, when posting from Python it never shows up. I did compare the requests for both and have not been able to spot a difference.
{
"hitParsingResult": [ {
"valid": true,
"parserMessage": [ ],
"hit": "/debug/collect?v=1\u0026t=event\u0026tid=XXXXXXX\u0026cid=YYYYYYu0026ec=Slack\u0026ea=SlashCommand\u0026el=whowasat-curl\u0026an=staging.Whereis-Everybody?\u0026aid=staging.whereis-everybody.com"
} ],
"parserMessage": [ {
"messageType": "INFO",
"description": "Found 1 hit in the request."
} ]
}
The NodeJS code is (result does show up in Google Analytics):
'use strict';
var request = require('request');
require('request-debug')(request);
function postEventToGA(category, action, label) {
var options = {
v: '1',
t: 'event',
tid: process.env.GOOGLEANALYTICS_TID,
cid: process.env.GOOGLEANALYTICS_CID,
ec: category,
ea: action,
el: label,
an: process.env.STAGE_INFIX + "appname",
aid: process.env.STAGE_INFIX + "appname"
};
console.log("payload: " + JSON.stringify(options))
request.post({ url: 'https://www.google-analytics.com/collect', form: options }, function (err, response, body) {
console.log(request)
if (err) {
console.log("Failed to post event to Google Analytics, error: " + err);
} else {
if (200 != response.statusCode) {
console.log("Failed to post event to Google Analytics, response code: " + response.statusCode + " error: " + err);
}
}
});
}
postEventToGA("some-category", "some-action", "some-label")
And the Python code is (result does not show up in Google Analytics):
import json
import logging
import os
import requests
LOGGER = logging.getLogger()
LOGGER.setLevel(logging.INFO)
GOOGLEANALYTICS_TID = os.environ["GOOGLEANALYTICS_TID"]
GOOGLEANALYTICS_CID = os.environ["GOOGLEANALYTICS_CID"]
STAGE_INFIX = os.environ["STAGE_INFIX"]
def post_event(category, action, label):
payload = {
"v": "1",
"t": "event",
"tid": GOOGLEANALYTICS_TID,
"cid": GOOGLEANALYTICS_CID,
"ec": category,
"ea": action,
"el": label,
"an": STAGE_INFIX + "appname,
"aid": STAGE_INFIX + "appname",
}
response = requests.post("https://www.google-analytics.com/collect", payload)
print(response.request.method)
print(response.request.path_url)
print(response.request.url)
print(response.request.body)
print(response.request.headers)
print(response.status_code)
print(response.text)
if response.status_code != 200:
LOGGER.warning(
"Got non 200 response code (%s) while posting to GA.", response.status_code
)
post_event("some-category", "some-action", "some-label")
Any idea why the NodeJS post will show up in Google Analytics and the Python post does not?
(while both return a HTTP200)
Did some more testing and discovered that the user agent HTTP header was causing the problem. When I set it to an empty string in the Python code it works. Like this:
headers = {"User-Agent": ""}
response = requests.post(
"https://www.google-analytics.com/collect", payload, headers=headers
)
The documentation at https://developers.google.com/analytics/devguides/collection/protocol/v1/reference does state that the user agent is used, but does not clearly state what the requirements are. "python-requests/2.22.0" (default value by python-requests lib) is apparently not accepted.
Related
I have python code that successfully downloads a Nessus scan report in csv format, but I need to add some additional fields to the downloaded report. I include parameters in the request payload to include some fields, but the scan that is downloaded does not include those fields.
I've tried changing the value of the reportedContents params to actual Boolean types with the True keyword.
Also, I changed the format to pdf and it exports a PDF file that is just a title page and a page with a blank table of contents.
The downloaded csv file has data in it, but only includes the default headers (i.e.):
Plugin ID,CVE,CVSS v2.0 Base Score,Risk,Host,Protocol,Port,Name,Synopsis,Description,Solution,See Also,Plugin Output
The raw output of the POST request looks like:
POST https://localhost:8834/scans/<scan_id>/export
X-ApiKeys: accessKey=accessKey;secretKey=secretKey
Content-Type: application/x-www-form-urlencoded
Content-Length: 122
format=csv&reportContents.vulnerabilitySections.exploitable_with=true&reportContents.vulnerabilitySections.references=true
def download_scan(scan_num):
# Post an export request
headers = {
'X-ApiKeys': 'accessKey=accessKey;secretKey=secretKey',
'Content-Type': 'application/x-www-form-urlencoded'
}
data = {
'format': 'csv',
'reportContents.vulnerabilitySections.exploitable_with': 'true',
'reportContents.vulnerabilitySections.references': 'true'
}
res = requests.post(url + '/scans/{id_num}/export'.format(id_num = scan_num), data=data, verify=False, headers=headers)
if res.status_code == 200:
export = json.loads(res.text)
file_id = export.get('file')
# Continually check the scan status until the status is ready
while True:
# Check file status
res = requests.get(url + '/scans/{id_num}/export/{file_num}/status'.format(id_num = scan_num, file_num = file_id), verify=False, headers=headers)
if res.status_code == 200:
status = json.loads(res.text)['status']
if status == 'ready':
break
# Download the scan
res = requests.get(url + '/scans/{scan_num}/export/{file_num}/download'.format(scan_num = scan_num, file_num = file_id), verify=False, headers=headers)
# If the scan is successfully downloaded, get the attachment file
if res.status_code == 200:
attachment = res.content
print("Scan downloaded!!!")
else:
raise Exception("Download request failed with status code: " + str(res))
return attachment
def main():
# Download the scan based on the scan_id. I have a helper function that returns the id that I am omitting here
try:
scan = download_scan(scan_id)
except Exception as e:
print(e)
quit()
with open("scan.csv", "wb") as f:
f.write(scan)
f.close()
if __name__ == "__main__":
main()
I'm having the exact same issue but with PowerShell. Neither my additional columns nor filters appear to be working. Was wondering if you'd had any joy getting this to work?
If I change the scan_id I get the correct different results, which suggests it is receiving the JSON but ignoring the columns and filters.
My JSON is as follows...
{
"scan_id": 3416,
"format": "csv",
"reportContents.vulnerabilitySections.cvss3_base_score": true,
"filters": {
"filter.0.quality": "gt",
"filter.0.filter": "cvss2_base_score",
"filter.0.value": "6.9",
"filter.1.quality": "neq",
"filter.1.filter": "cvss2_base_score",
"filter.1.value": ""
}
}
I managed to fix it, my problem was that I was using Python's requests module and it's data={} keyword, which defaults to header content-type: application-x-www-form-urlencoded, it generates reports with strictly 13 fields regardless of your payload.
To make it actually consider your payload, use the header "content-type": "application/json", in your code implicitly and json={} in your payload instead of data={}.
WILL NOT WORK:
requests.post(
nessus_url + f"/scans/{scan_id}/export",
data={
"format": "csv",
"template_id": "",
"reportContents": {
"csvColumns": {
"id": True,
"cve": True,
"cvss": True,
**other_columns,
}
}
},
verify=False,
headers={
"X-ApiKeys": f"accessKey={credentials['access_key']}; secretKey={credentials['secret_key']}",
},
)
WILL WORK:
requests.post(
nessus_url + f"/scans/{scan_id}/export",
json={
"format": "csv",
"template_id": "",
"reportContents": {
"csvColumns": {
"id": True,
"cve": True,
"cvss": True,
**other_columns
}
}
},
verify=False,
headers={
"X-ApiKeys": f"accessKey={credentials['access_key']}; secretKey={credentials['secret_key']}",
"content-type": "application/json",
},
)
I'm looking to convert this Python request to a Swift script.
Here is my working python script that returns the accessToken!
#!/usr/bin/python
import requests
import json
#MAKE THE REQUEST
URL = "http://this/is/the/url"
headers = {
'Accept': "application/json",
"Accept-Language": "en_US"
}
data = {
"grant_type": "password",
"username" : "GROUP\SITE\USERNAME",
"password" : "somepassword"
}
r = requests.get(url = URL, params = headers, data = data)
data = r.json()
accessToken = data['access_token']
print(accessToken)
When I run the Swift Playground for the code below nothing is returned!
It seems the script exits at guard let data = data else { return }
How could I get the same results as the Python Script above.
I've tried implementing URLComponents using this tutorial...
import UIKit
var url = "http://just/the/url"
extension Dictionary {
func percentEncoded() -> Data? {
return map { key, value in
let escapedKey = "\(key)"
let escapedValue = "\(value)"
print(escapedKey + "=" + escapedValue)
return escapedKey + "=" + escapedValue
}
.joined(separator: "&")
.data(using: .utf8)
}
}
extension CharacterSet {
static let urlQueryValueAllowed: CharacterSet = {
let generalDelimitersToEncode = ":#[]#" // does not include "?" or "/" due to RFC 3986 - Section 3.4
let subDelimitersToEncode = "$&'()*+,;="
var allowed = CharacterSet.urlQueryAllowed
allowed.remove(charactersIn: "\(generalDelimitersToEncode)\(subDelimitersToEncode)")
return allowed
}()
}
var request = URLRequest(url: URL(string:url)!)
request.httpMethod = "GET"
let parameters: [String: String] = [
"grant_type":"password",
"username":"GROUP\\SITE\\USER",
"password":"somePassword"
]
request.httpBody = parameters.percentEncoded()
request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type")
request.setValue("application/XML", forHTTPHeaderField: "Accept")
let config = URLSessionConfiguration.default
URLSession(configuration: config).dataTask(with: request) { (data, response, err) in
guard let data = data else { return }
print(data)
guard let dataAsString = String(data: data, encoding: .utf8)else {return}
print(dataAsString)
guard let httpResponse = response as? HTTPURLResponse,
(200...299).contains(httpResponse.statusCode) else {
print("Bad Credentials")
return
}
//HTTP Status Code!
print("HTTP RESPONSE:"+"\(httpResponse.statusCode)")
//
}.resume()
If I remember correctly, starting in iOS 13, you cant have httpBody for a GET call, so you'll either need to switch to a POST/PUT or add the params into the url string (See below)
You also had different Accept headers in your python vs. swift. One was xml the other was json.
var urlComponents = URLComponents(string: "http://this/is/the/url")
urlComponents?.queryItems = [
URLQueryItem(name: "grant_type", value: "password"),
URLQueryItem(name: "username", value: "username"),
URLQueryItem(name: "password", value: "somepassword")
]
guard let url = urlComponents?.url else { return } // You can print url here to see how it looks
var request = URLRequest(url: url)
request.httpMethod = "GET"
request.setValue("application/json", forHTTPHeaderField: "Accept")
request.setValue("en_US", forHTTPHeaderField: "Accept-Language")
let task = URLSession.shared.dataTask(with: request) { data, response, error in
guard let data = data,
let response = response as? HTTPURLResponse,
error == nil else {
print("error", error ?? "Unknown error")
return
}
print(response)
guard (200 ... 299) ~= response.statusCode else {
print("response = \(response)")
return
}
let responseString = String(data: data, encoding: .utf8)
print(responseString)
}
task.resume()
The problem was the following...
request.httpMethod = "GET"
I had to change the get to "POST" and now I have the token!!!!
I was confused because the python script used GET. I had a bash script that that used curl to get the token displayed the logged post.
In short my above Swift Playground now works by changing the request.httpMethod to "POST". THANKS FOR ALL THE HELP
I have created a Facebook Messenger bot which works fine. I have used the Button Template and Image template, and both work perfectly. But when I try the Generic Template, I get no response. I have simply copy pasted the code from here, by performing the appropriate modifications.
I don't know how to debug. Facebook Messenger gives no output on the messaging box. I am currently running the app via Heroku.
Here is my code:
def send_message(token, recipient):
r = requests.post("https://graph.facebook.com/v2.6/me/messages",
params={"access_token": token},
data=json.dumps({
"recipient":{
"id":recipient
},
"message":{
"attachment":{
"type":"template",
"payload":{
"template_type":"generic",
"elements":[
{
"title":"Welcome to Peter\'s Hats",
"image_url":"http://www.godominion.com/content/images/feature-img-small-appliance-electronics.png",
"subtitle":"We\'ve got the right hat for everyone.",
"default_action": {
"type": "web_url",
"url": "https://peterssendreceiveapp.ngrok.io/view?item=103",
"messenger_extensions": true,
"webview_height_ratio": "tall",
"fallback_url": "https://peterssendreceiveapp.ngrok.io/"
},
"buttons":[
{
"type":"web_url",
"url":"https://petersfancybrownhats.com",
"title":"View Website"
}
]
}
]
}
}
}
}),
headers={'Content-type': 'application/json'})
if r.status_code != requests.codes.ok:
print r.text
I would appreciate any help.
Thank you.
EDIT 1: SOLUTION
I got rid of the issue by commenting out:
"messenger_extensions": true,
and
"fallback_url": "https://peterssendreceiveapp.ngrok.io/"},
I'm sure this is not the correct method. But as I am creating a bot, without actual links, this works.
On the second button, "url":"https://petersfancybrownhats.com" is broken.
Try like this
firstly make a function
def function():
extra_data = {
"attachment": {
"type": "template",
"payload": {
"template_type": "generic",
"elements": [
{
"title": "Any Title",
"image_url": "https://mbtskoudsalg.com/images/road-clipart-journey-3.png",
"subtitle": "Subtitle.",
"buttons": [
{
"type": "web_url",
"title": "View",
"url": "**MAKE SURE TO WHITELIST THIS URL**", # URL
"messenger_extensions": "true",
"webview_height_ratio": "full"
}
]
}
]
}
}
}
# w_message = "Hi there! How may I help you?"
fb_message_template(extra_data["attachment"], "****RECIEVER ID****")
Make another function
import requests
# // Importing User Defined Modules // #
from get_environ_var import get_environ_var
# // Global vars // #
ACCESS_TOKEN = "FB_ACCESS_TOKEN"
def fb_message_template(extra_data, sender_id):
"""This function sends template message to facebook"""
data = {
'recipient': {'id': sender_id},
'message': {
"attachment": extra_data
}
}
qs = 'access_token=' + ACCESS_TOKEN
resp = requests.post('https://graph.facebook.com/v2.6/me/messages?' + qs, json=data)
print(resp.content)
I'm trying to follow some examples from Python Facebook Marketing Api but, when I run:
i_async_job = account.get_insights(params={'level': 'adgroup'}, async=True)
r_async_job = account.get_report_stats(
params={
'data_columns': ['adgroup_id'],
'date_preset': 'last_30_days'
},
async=True
)
I'm getting
Status: 400
Response:
{
"error": {
"message": "(#12) adaccount/reportstats is deprecated for versions v2.4 and higher",
"code": 12,
"type": "OAuthException"
}
}
Even from Facebook
I found this page, but there are only curl examples.
Is there a working example on how to get data from Insights edge with the Python Ads API?
Here is a full example of how to export some insights asynchronously from the new Insights endpoints:
from facebookads import test_config as config
from facebookads.objects import *
import time
account_id = <YOUR_ACCOUNT_ID>
account_id = 'act_' + str(account_id)
fields = [
Insights.Field.impressions,
Insights.Field.clicks,
Insights.Field.actions,
Insights.Field.spend,
Insights.Field.campaign_group_name,
]
params = {
'date_preset': Insights.Preset.last_7_days,
'level': Insights.Level.adgroup,
'sort_by': 'date_start',
'sort_dir': 'desc',
}
ad_account = AdAccount(account_id)
job = ad_account.get_insights(fields=fields, params=params, async=True)
insights = None
while insights is None:
time.sleep(1)
job.remote_read()
completition = job[AsyncJob.Field.async_percent_completion]
print("Percent done: " + str(completition))
if int(completition) is 100:
insights = job.get_result(params={'limit': 100})
for ad_insight in insights:
print(ad_insight)
The example code for Google's YouTube Data API is a piece of junk. It's so complicated and tied to the oauth redirect flow that I can't use it. Trying to go raw with requests pip and not getting too far.
I've followed the instructions exactly (as far as I can tell), with the following code:
import json
import os
import sys
import urllib
import requests
payload_file = None
payload = None
print 'Loading Config'
# Get the directory path of this file. When using any relative file paths make
# sure they are relative to current_dir so that the script can be run from any CWD.
current_dir = os.path.dirname(os.path.abspath(__file__))
# Reads in the config.json file then parses it
config = json.loads(open(os.path.join(current_dir, '..', 'config.json')).read())
print 'Parsing Payload'
for i in range(len(sys.argv)):
if sys.argv[i] == "--json" and (i + 1) < len(sys.argv):
payload = json.loads(sys.argv[i + 1])
elif sys.argv[i] == "-payload" and (i + 1) < len(sys.argv):
payload_file = sys.argv[i + 1]
with open(payload_file,'r') as f:
payload = json.loads(f.read())
break
print 'Configuring youtube with token {0}'.format(payload['token'])
print 'Downloading video...'
# See how big it is
f = urllib.urlopen(payload['url'])
content_length = int(f.headers["Content-Length"])
# Download it
# urllib.urlretrieve(payload['url'], "video.mp4")
metadata = {
'snippet' : {
'title': payload['title'],
"categoryId": 22
},
'status' : {
"privacyStatus": "public",
"embeddable": True,
"license": "youtube"
}
}
if 'tags' in payload:
metadata['snippet']['tags'] = payload['tags']
if 'description' in payload:
metadata['snippet']['description'] = payload['description']
headers = {
'Authorization' : 'Bearer {0}'.format(payload['token']),
'Content-Type' : 'application/json; charset=UTF-8',
'Content-Length' : json.dumps(metadata).__len__(),
'X-Upload-Content-Length' : content_length,
'X-Upload-Content-Type' : 'video/*',
}
print 'Attempting to upload video'
print headers
# upload video file
r = requests.post('https://www.googleapis.com/upload/youtube/v3/videos?uploadType=resumable&part=snippet,status', data=metadata, headers=headers);
print "RESPONSE!"
print r.text
# files = {
# 'file': video_file,
# }
# r = requests.post('https://www.googleapis.com/upload/youtube/v3/videos', data={ "video" : video }, headers=headers);
Obviously its not finished, but its dying on the metadata upload request with the following output:
Loading Config
Parsing Payload
Configuring youtube with token <access-token>
Downloading video...
Attempting to upload video
{'X-Upload-Content-Length': 51998563, 'Content-Length': 578, 'Content-Type': 'application/json; charset=UTF-8', 'X-Upload-Content-Type': 'video/*', 'Authorization': 'Bearer <access-token>'}
RESPONSE!
{
"error": {
"errors": [
{
"domain": "global",
"reason": "parseError",
"message": "Parse Error"
}
],
"code": 400,
"message": "Parse Error"
}
}
This error is not even listed in their "Errors" docs.
What is wrong with my code?
Here is an example in python that works. It assumes you've already done the oauth part though.
import requests
from os import fstat
import json
fi = open('myvideo.mp4')
base_headers = {
'Authorization': '%s %s' % (auth_data['token_type'],
auth_data['access_token']),
'content-type': 'application/json'
}
initial_headers = base_headers.copy()
initial_headers.update({
'x-upload-content-length': fstat(fi.fileno()).st_size,
'x-upload-content-type': 'video/mp4'
})
initial_resp = requests.post(
'https://www.googleapis.com/upload/youtube/v3/videos?uploadType=resumable&part=snippet,status,contentDetails',
headers=initial_headers,
data=json.dumps({
'snippet': {
'title': 'my title',
},
'status': {
'privacyStatus': 'unlisted',
'embeddable': True
}
})
)
upload_url = initial_resp.headers['location']
resp = requests.put(
upload_url,
headers=base_headers,
data=fi
)
fi.close()
the above is graet, just adding: you can also get the youtube id from the response (for future use):
cont = json.loads(resp.content)
youtube_id = cont['id']