I have a file which was exported from BIND containing TSIG values for about 500 domain names. I need to repurpose the data into JSON for a REST API query. The BIND data is formatted like so:
// secondary-example.com.
key "2000000000000.key." {
algorithm hmac-md5;
secret "ahashedvalue=";
};
zone "secondary-example.com." {
type slave;
file "sec/secondary-example.com.";
allow-transfer { 1.1.1.1;
1.1.2.2;
};
also-notify { 1.1.1.1;
2.2.2.2;
};
masters {
1.2.3.4 key 2000000000000.key.;
};
};
From this I need to extract the key, zone and secret. Here's an example API request.
{
"properties":{
"name":"secondary-example.com.",
"accountName":"example",
"type":"SECONDARY"
},
"secondaryCreateInfo":{
"primaryNameServers":{
"nameServerIpList":{
"nameServerIp1":{
"ip":"1.2.3.4",
"tsigKey":"2000000000000.key.",
"tsigKeyValue":"ahashedvalue="
}
}
}
}
}
I'm having difficulty crafting a regular expression appropriate for the scenario. I'm looking construct the JSON in a python script and send the request through Postman.
I spent a couple days reading up on regex and figured out a solution. So, each of those "zones" began with a comment... e.g. "secondary-example.com"... and each set of BIND info was 17 lines long exactly. This solution is hackey and always assumes data is correct, but it managed to work.
Separate the zones into chunks of text.
zones = []
cur_zone = ''
f = open(bind_file).readlines()
for line in f:
if line[0:2] == '//':
zones.append(cur_zone)
cur_zone = ''
else:
cur_zone = cur_zone + line
zones.pop(0) # Drop the first list item, it's empty
Iterate through those chunks and match the needed parameters.
for z in zones:
z_lines = z.splitlines()
# Regex patterns to make the required parameters
key = re.findall('\"(.*)\"', z_lines[0])[0]
secret = re.findall('\"(.*)\"', z_lines[2])[0]
name = re.findall('\"(.*)\"', z_lines[5])[0]
master = re.findall('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', z_lines[15])[0]
Related
So I'm working with AEM and am attempting to create a script that grabs all pages under a specific path and updates the image reference on the page from a list of assets under a curtain path.
Both of my select query's aren't returning the specific pages and assets I need.
I'm also getting an error that my queries are searching over 100000 Nodes
How can i resolve this error and query my resources better?
import com.day.cq.wcm.api.Page
import javax.jcr.query.Query
import javax.jcr.query.QueryManager
import org.apache.sling.api.resource.ModifiableValueMap
import groovy.transform.Field
static void main(String[] args)
{
String[] assetNodes
String[] pageNodes
String pagePath ="/content/we-retail/us/en"
String pageResourceType = "weretail/components/structure/page"
String assetPath ="/content/dam/microsoft/internal/en"
String assetQuery = "b1048291-23fa-422a-a7c4-9ea4bae0effc"
boolean isAsset = true;
pageNodes = GetResourcePath(pagePath, pageResourceType);
assetNodes = GetRosourceAsset(assetPath, assetQuery);
InputAssetsOnPage(pageNodes,assetNodes);
}
//Find the Node paths for all Pages to modify
//Narrow down to image component
def GetResourcePath(String rootPath,String queryParam)
{
int i = 0
def String[] allNodes = new String[500]
Page page = getPage(rootPath)
def queryManager = session.workspace.queryManager;
def param= queryParam;
def statement = 'select * from nt:base where jcr:path like \''+page.path+'/%\' and sling:resourceType = \'' + param + '\'';
Query query=queryManager.createQuery(statement, 'sql');
final def result = query.execute()
println "Total pages found = " + result.nodes.size();
NodeIterator nodeIterator = result.getNodes();
while(nodeIterator.hasNext())
{
def hitNode = nodeIterator.nextNode();
allNodes[i] = hitNode.getPath();
i++;
}
println allNodes
return allNodes;
}
//Find all assets paths to add to page
def GetRosourceAsset(String rootPath,String queryParam)
{
int i = 0
def String[] allNodes = new String[500]
Page page = getPage(rootPath)
def queryManager = session.workspace.queryManager;
def param= queryParam;
def statement = 'select * from nt:base where jcr:path like \''+rootPath+'/%\'';
Query query=queryManager.createQuery(statement, 'sql');
final def result = query.execute()
println "Total Assets found = " + result.nodes.size();
NodeIterator nodeIterator = result.getNodes();
while(nodeIterator.hasNext())
{
def hitNode = nodeIterator.nextNode();
allNodes[i] = hitNode.getPath();
i++;
}
println allNodes
return allNodes;
}
//Modify image component property with unique asset path
void InputAssetsOnPage(String[]pageRefrence, String[]assetRefrences)
{
String[] nodes= pageRefrence;
String[] assetNodes = assetRefrences;
nodes.eachWithIndex { self,i->
javax.jcr.Node node=getNode(nodes[i])
ModifiableValueMap mVMap=resourceResolver.resolve(node.path).adaptTo(ModifiableValueMap.class);
mVMap.put("fileRefrence", assetNodes[i]);
println "Property modified to "+node.path;
println "Dry Run "+data.dryRun;
if(!data.dryRun) {
session.save();
}
}
}
//Save session
For mass-updates it is very likely to target too many nodes. You have to try some approaches, to either get just under the limit - or change you approach.
First select pages from cq:PageContent, instead from nt:base. The query-indexes are organised by jcr:primaryType. The nt:base index contains everything. So there are much more nodes.
Second use SQL-2 and ISDESCENDANTNODE() instead of a like operator. I don't expect the like-operator to be so specific. But, if you query almost all pages anyway - it won't help much.
Third iterate over parts of you page-tree. Then the remaining subtree is much small, can can be queried.
For a mass-update, which probably touches more than 15% of your pages, then just iterate over all pages. Don't forget to commit from time to time. (e.g. every 100 changes)
Code sample to iterate a page-tree
def rootPage = getPage("/content/uac/glooly/es")
rootPage.recurse() { page ->
println(page.path)
fixAssetsOnPage(page)
}
You can iterate over the first n levels over the content tree, and then deep dive with a query. So you could have less than 100.000 image components
Code sample to iterate over first 2 levels, and then query:
def rootPage = getPage("/content/we-retail/us/en")
rootPage.iterator().each { firstLevelPage ->
firstLevelPage.iterator().each { secondLevelPage ->
println(secondLevelPage.path)
queryAndFixAssets(secondLevelPage)
save()
}
}
Fourth create an oak:index for you query. Especially if you could use it for later too. But in your case there is an existing index for sling:resourceType, which should be could enough. You have just too many hits.
PS: You probably don't need to query pages. Just query the image components in a small enough subtree. But we can't see that from you code sample.
I am trying to build an IOT setup. I am thinking of using a json file to store states of the sensors and lights of the setup.
I have created a function to test out my concept. Here is what I wrote so far for the data side of things.
{
"sensor_data": [
{
"sensor_id": "302CEM/lion/light1",
"sensor_state": "on"
},
{
"sensor_id": "302CEM/lion/light2",
"sensor_state": "off"
}
]
}
def read_from_db():
with open('datajson.json') as f:
data = json.load(f)
for sensors in data['sensor_data']:
name = sensors['sensor_id']
read_from_db()
What I want to do is to parse the sensor_id into an array so that I can access them by saying for example sensor_name[0]. I am not sure how to go about it. I tried array.array but it doesn't save any values, have also tried .append but not the result I expected. Any suggestions?
If I understood correctly, all you have to do is assign all those sensors to names using a for loop and then return the result:
import json
def read_from_db():
with open('sensor_data.json') as f:
data = json.load(f)
names = [sensors['sensor_id'] for sensors in data['sensor_data']]
return names
sensor_names = read_from_db()
for i in range(len(sensor_names)):
print(sensor_names[i])
This will print:
302CEM/lion/light1
302CEM/lion/light2
I'm using Django Rest Framework as my backend and Angular 2 for my frontend. I've got this page in Angular, where I create a form:
createResourcesForm() {
this.resourcesForm = this.formBuilder.group({
resources: this.formBuilder.array([
this.formBuilder.group({
title: ['', Validators.compose([Validators.required])],
file: ['', Validators.compose([])],
})
])
})
}
As you can see, the form consists of FormArray, where every element has two inputs: title and file - a text input and a file input respectively. On submitting the form I'm trying to send the data to Django but I get an error Missing filename. Request should include a Content-Disposition header with a filename parameter.. I could set it easily but I'm expecting to receive a list of {title, file}, so how to set multiple file names? Any other idea how I could do this?
The error in Django Rest Framework comes from parse method in FileUploadParser.
I'm not pasting any Python code here because it's a standard ListCreateAPIView, nothing special about it. Here is my serializer:
class ResourceCreateSerializer2(serializers.Serializer):
author_pk = serializers.IntegerField(required=False)
author_first_name = serializers.CharField(max_length=50, required=False)
author_last_name = serializers.CharField(max_length=50, required=False)
resources = ResourceWithoutAuthorSerializer(many=True)
class ResourceWithoutAuthorSerializer(serializers.ModelSerializer):
instruments = InstrumentSerializer(many=True)
class Meta:
model = MusicResource
fields = ['title', 'file', 'instruments']
Don't mind the other fields, they are being sent just fine (as the file does). One more thing to say - I'm adding a content-type header in Angular just before sending the data.
UPDATE 1
Here is my method for uploading files (Angular 2):
get value() {
let formData = new FormData();
for (let i = 0; i < this.resources.length; i++) {
let resource = this.resources.value[i];
let fileName = resource.file;
let fileInputs = $('input[type="file"]').filter(function () {
return this.value == fileName;
});
if (fileInputs.length == 0) {
return null;
}
let fileInput = <HTMLInputElement>fileInputs[0];
formData.append('resources-' + i + '-title', resource.title);
formData.append('resources-' + i + '-file', fileInput.files[0], fileInput.files[0].name);
for (let j = 0; j < this.instrumentsSelect.value.length; j++) {
formData.append('resources-' + i + '-instruments', this.instrumentsSelect.value[j]);
}
}
return formData;
}
then
this.musicResourcesService.addMusicResource(toSend).subscribe(
data => console.log('successfuly added resources'),
err => console.log('MusicResourcesAddComponent', 'onMusicResourceFormSubmit', err)
);
addMusicResource(data) {
let headers = new Headers({});
headers.append('Content-Type', 'multipart/form-data');
headers.append('Accept', 'application/json');
let options = new RequestOptions({headers});
return this.api.post('resources/resources/list_create/', data, true, options);
}
public post(url: any, payload: any, noToken?, options?: any): Observable<any> {
const provider = noToken ? this.http : this.authHttp;
const fulLUrl = this.conf.getAPIUrl() + url;
return provider.post(fulLUrl, payload, options)
.delay(100)
.map(this.extractData)
.catch(this.handleError).share();
}
I did not like #Robert's answer and did not receive any other idea so after hours of reseraching it turns out that I was missing two things:
The parser should have been set to MultiPartParser, not FileUploadParser
There is no need to set the Content-Type header manually, it will get filled automatically along with the boundary which I was missing
Also, to make sure Django receives all the data and understands it, I had to change
formData.append('resources-' + i + '-title', resource.title);
and similar lines to
formData.append('resources[' + i + ']title', resource.title);
Basically what I am trying to do is generate a json list of SSH keys (public and private) on a server using Python. I am using nested dictionaries and while it does work to an extent, the issue lies with it displaying every other user's keys; I need it to list only the keys that belong to the user for each user.
Below is my code:
def ssh_key_info(key_files):
for f in key_files:
c_time = os.path.getctime(f) # gets the creation time of file (f)
username_list = f.split('/') # splits on the / character
user = username_list[2] # assigns the 2nd field frome the above spilt to the user variable
key_length_cmd = check_output(['ssh-keygen','-l','-f', f]) # Run the ssh-keygen command on the file (f)
attr_dict = {}
attr_dict['Date Created'] = str(datetime.datetime.fromtimestamp(c_time)) # converts file create time to string
attr_dict['Key_Length]'] = key_length_cmd[0:5] # assigns the first 5 characters of the key_length_cmd variable
ssh_user_key_dict[f] = attr_dict
user_dict['SSH_Keys'] = ssh_user_key_dict
main_dict[user] = user_dict
A list containing the absolute path of the keys (/home/user/.ssh/id_rsa for example) is passed to the function. Below is an example of what I receive:
{
"user1": {
"SSH_Keys": {
"/home/user1/.ssh/id_rsa": {
"Date Created": "2017-03-09 01:03:20.995862",
"Key_Length]": "2048 "
},
"/home/user2/.ssh/id_rsa": {
"Date Created": "2017-03-09 01:03:21.457867",
"Key_Length]": "2048 "
},
"/home/user2/.ssh/id_rsa.pub": {
"Date Created": "2017-03-09 01:03:21.423867",
"Key_Length]": "2048 "
},
"/home/user1/.ssh/id_rsa.pub": {
"Date Created": "2017-03-09 01:03:20.956862",
"Key_Length]": "2048 "
}
}
},
As can be seen, user2's key files are included in user1's output. I may be going about this completely wrong, so any pointers are welcomed.
Thanks for the replies, I read up on nested dictionaries and found that the best answer on this post, helped me solve the issue: What is the best way to implement nested dictionaries?
Instead of all the dictionaries, I simplfied the code and just have one dictionary now. This is the working code:
class Vividict(dict):
def __missing__(self, key): # Sets and return a new instance
value = self[key] = type(self)() # retain local pointer to value
return value # faster to return than dict lookup
main_dict = Vividict()
def ssh_key_info(key_files):
for f in key_files:
c_time = os.path.getctime(f)
username_list = f.split('/')
user = username_list[2]
key_bit_cmd = check_output(['ssh-keygen','-l','-f', f])
date_created = str(datetime.datetime.fromtimestamp(c_time))
key_type = key_bit_cmd[-5:-2]
key_bits = key_bit_cmd[0:5]
main_dict[user]['SSH Keys'][f]['Date Created'] = date_created
main_dict[user]['SSH Keys'][f]['Key Type'] = key_type
main_dict[user]['SSH Keys'][f]['Bits'] = key_bits
I have a datastore kind "Items" which was created in Python, in this code do not iterate data q.Run() in Go (it's version 2):
type Items struct{
code string
date time.Time
name string
}
func getcode(w http.ResponseWriter, r *http.Request) {
code := mux.Vars(r)["code"]
fmt.Fprintf(w,"get code %v",code)
c := appengine.NewContext(r)
q := datastore.NewQuery("Items")
for t := q.Run(c); ; {
var x Items
key, err := t.Next(&x)
fmt.Fprintf(w,"%v",key)
if err == datastore.Done {
break
}
if err != nil {
//serveError(c, w, err)
return
}
fmt.Fprintf(w, "Code=%v\n", x.code)
}
The Datastore package uses reflection to fill struct fields when reading an entity from the datastore. In Go struct fields whose name start with lowercase letter are not exported. Unexported fields cannot be set from packages other than the one they were defined in.
Only exported fields (that start with uppercase letters) can be stored in / retrieved from the datastore. You can use tags to tell what the name of the property is in the datastore in case it differs from the field's name. So you have to change your Items struct to this:
type Items struct {
Code string `datastore:"code"`
Date time.Time `datastore:"date"`
Name string `datastore:"name"`
}