Blender export addon and bl_idname

Blender export addon and bl_idname - python

I'm trying to write a simple exporter for Blender in Python and I think the issue is with bl_idname. I'm not sure exactly how the value should be formatted.
class ExportS3D(bpy.types.Operator, ExportHelper) :
bl_idname = "object.ExportS3D";
bl_label = "S3D Exporter";
bl_options = {'PRESET'};
filename_ext = ".S3D";
I could be completely wrong on what I'm doing wrong, so here's my code:
bl_info = {
"name": "S3D Exporter",
"author": "M---",
"blender": (2,7,1),
"version": (0,0,1),
"location": "File > Import-Export > S3D",
"description": "Export S3D files",
"category": "Import-Export"
}
import bpy
from bpy_extras.io_utils import ExportHelper
import time
class ExportS3D(bpy.types.Operator, ExportHelper) :
bl_idname = "object.ExportS3D";
bl_label = "S3D Exporter";
bl_options = {'PRESET'};
filename_ext = ".S3D";
def execute(self, context):
export()
#end ExportS3D
def menu_func(self, context):
self.layout.operator(object.bl_idname, text="Stupid 3D(.S3D)");
def register():
bpy.utils.register_module(__name__);
bpy.types.INFO_MT_file_export.append(menu_func);
def unregister():
bpy.utils.unregister_module(__name__);
bpy.types.INFO_MT_file_export.remove(menu_func);
def export():
print( '\n--------------------\n' )
#--------------------------------------
#Change to OBJECT mode
#Do this to ensure any changes made in EDIT mode (like UV Unwrap) are committed
bpy.ops.object.mode_set( mode='OBJECT' )
#--------------------------------------
#Get the active object and its data
ob = bpy.context.active_object
if ( ob.type != 'MESH' ):
print( 'Error: please select a MESH object.\n' )
return
mesh = ob.data
#Not sure if this is needed with this script???
if not mesh.tessfaces and mesh.polygons:
mesh.calc_tessface()
#--------------------------------------
#Comments
comments = ''
localtime = time.localtime()
timeString = time.strftime( '%B %d, %Y %H:%M:%S', localtime )
comments += ( '%s\n' % (timeString) )
comments += ( '3D object created in Blender%s\n' % (bpy.app.version_string) )
comments += ( 'Object name: %s\n' % (ob.name) )
comments += ( 'Blender vertices count: %i\n' % ( len(mesh.vertices) ) )
comments += ( 'Blender tessfaces count: %i\n' % ( len(mesh.tessfaces) ) )
#--------------------------------------
#UV Layer
if ( mesh.uv_layers.active is not None ):
uv_layer = mesh.uv_layers.active.data
else:
print( 'Error: the object needs to be unwrapped.\n' )
return
#--------------------------------------
#Vertices and Indices
vertices = 'Vertices\n';
indices = 'Indices\n';
image = 'Image Name\n';
i = 0
t = 0
c = 0
for poly in mesh.polygons:
for loop_index in poly.loop_indices:
v = mesh.vertices[mesh.loops[loop_index].vertex_index]
#Right-handed coordinate systems (OpenGL convention) use: v.co.z and v.normal.z
#Left-handed coordinate systems (DirectX convention) use: -1*v.co.z and -1*v.normal.z
#OpenGL textures use: uv_layer[loop_index].uv[1]
#DirectX textures use: 1-uv_layer[loop_index].uv[1]
vertices += (
'%f, %f, %f, %f, %f, %f, %f, %f\n' % \
(
v.co.x, v.co.y, v.co.z,
v.normal.x, v.normal.y, v.normal.z,
uv_layer[loop_index].uv[0], uv_layer[loop_index].uv[1],
)
)
c += 1
#OpenGL convention is counter-clockwise winding.
#DirectX convention is clockwise winding.
if ( len(poly.vertices) == 3 ):
#clockwise winding:
#indices += ( '%i, %i, %i, ' % ( i, i+2, i+1 ) )
#counter-clockwise winding:
indices += ( '%i, %i, %i, ' % ( i, i+1, i+2 ) )
i += 3
t += 1
elif ( len(poly.vertices) == 4 ):
#clockwise winding:
#indices += ( '%i, %i, %i, ' % ( i, i+2, i+1 ) )
#indices += ( '%i, %i, %i, ' % ( i, i+3, i+2 ) )
#counter-clockwise winding:
indices += ( '%i, %i, %i, ' % ( i, i+1, i+2 ) )
indices += ( '%i, %i, %i, ' % ( i, i+2, i+3 ) )
i += 4
t += 2
else:
print( 'Error: faces with less than 3 or more than 4 vertices found.\n' )
return
image += '%s' % (bpy.context.object.data.uv_textures.active.data[0].image.name)
#Remove indices last comma and space
indices = indices[:-2]
comments += ( 'Exported vertices: %i\n' % ( i ) )
comments += ( 'Exported triangles: %i\n' % ( t ) )
comments += ( 'Exported indices: %i\n\n' % ( t * 3 ) )
comments += 'Format\nVertex: px, py, pz, nx, ny, nz, u, v\nIndices: i'
#--------------------------------------
#Write File
filenameSuffix = time.strftime( '%Y%d%m%H%M%S', localtime )
#File path
filenameFull = ( 'c:/Users/1043468/Desktop/%s.%s.S3D' % ( ob.name, filenameSuffix ) )
out_file = open( filenameFull, 'wt' )
out_file.write( '%s\n\n%s\n%s\n%s\n' % ( comments, vertices, indices, image) )
out_file.close()
print( '%s\n\nCompleted: %s\n' %(comments, filenameFull) )
if __name__ == "__main__":
register()
The filename is S3D_Eporter.py and it's placed in Blender/2.7.1/scripts/addons/

bl_idname defines the name used to access the operator within blender. Using "object.ExportS3D" your import operator is available as bpy.ops.object.ExportS3D except teh bl_idname needs to be lower case, so use "object.exports3d"
To get your importer to start working change register_module(__name__) to register_class(ExportS3D) same for unregister.
You will need to add return {'FINISHED'} to execute() and you can get rid of all the ';' you have.

Related

Fully monotone interpolation in python

I have some data such as:
I would like to fit a differentiable monotone curve to it. I tried PchipInterpolator class but on a very similar graph it resulted in:
This is not monotone.
How can I fit a monotone curve to data like this?
Here is a sample set of y values for another similar graph:
[0.1109157119023644, 0.20187393816931934, 0.14466318670239758, 0.16535159414166822, 0.05452708697483864, 0.2153046237959556, 0.2200300476272603, 0.21012762463269324, 0.15947100322395022, 0.2819691842129948, 0.15567770052985092, 0.24850595803020692, 0.1329341593280457, 0.15595107081606913, 0.3232021121832229, 0.23707961921686588, 0.2415887076540357, 0.32363506549779797, 0.3584089204036798, 0.29232772580068433, 0.22145994836140775, 0.22797587985241133, 0.2717787840603025, 0.3245255944762287, 0.29301098282789195, 0.32417076823344143, 0.3450906550996232, 0.34272097408024904, 0.3868714875012437, 0.41876692320045755, 0.3544198724867363, 0.33073960954801895, 0.3921033666371904, 0.33349050060172974, 0.3608862044547096, 0.37375822841635425, 0.5396399750708429, 0.4209201143798284, 0.42004773793166883, 0.5217725632679073, 0.5911731474218788, 0.43389609315065386, 0.4287288396176006, 0.43007525393257007, 0.5687062142675405, 0.6030811498722173, 0.5292225577714743, 0.47710974351051355, 0.6182720730381119, 0.6241033581931327, 0.6236788197617511, 0.6643161356364049, 0.5577616524049582, 0.6888440258481371, 0.6867893120660341, 0.6685257606057502, 0.599481675493677, 0.7309075091448749, 0.7644365338580481, 0.6176797601816733, 0.6751467827192018, 0.6452178017908761, 0.6684778262246701, 0.7003380077556168, 0.667035916425416, 0.8434451759113093, 0.8419343615815968, 0.8657695361433773, 0.7392487161484605, 0.8773282098364621, 0.8265679895117846, 0.7246599961191632, 0.7251899061730714, 0.9271640780410231, 0.9180581424305536, 0.8099033021701689, 0.8268585329594615, 0.8519967080830176, 0.8711231413093845, 0.8689802343798663, 0.8299523829217353, 1.0057741699770046, 0.8538130788729608, 0.9662784297225102, 1.023419780920539, 0.913146849759822, 0.9900885996579213, 0.8740638988529978, 0.8900285618419457, 0.9065474574434158, 1.0749522597307315, 1.0319120938258166, 1.0051369663172995, 0.9893558841613622, 1.051384986916457, 1.0327996870915341, 1.0945543972861898, 0.9716604944496021, 1.1490370559566179, 1.1379231481207432, 1.6836433783615088, 1.8162068766097395, 2.072155286917785, 2.0395966998366, 2.191064589600466, 2.1581974932543617, 2.163403843819597, 2.133441151300847, 2.1726053994136922, 2.1157865673629526, 2.2249636455682866, 2.2313062166802147, 2.1731708496472764, 2.315203950110816, 2.1601242661726827, 2.174940281421225, 2.2653635413275945, 2.337227057574145, 2.3645767548381618, 2.3084919291392527, 2.314014515926446, 2.25166717296155, 2.2621157708115778, 2.2644578546265586, 2.313504860292943, 2.398969190357051, 2.309443951779675, 2.278946047410807, 2.4080802287121146, 2.353652872018618, 2.35527529074088, 2.4233001060410784, 2.428767198055608, 2.35677123091093, 2.497135132404064, 2.3978099128437282, 2.3970802609341972, 2.4967434818740024, 2.511209192435555, 2.541001050440798, 2.5760248002036525, 2.5960512284192245, 2.4778408861721037, 2.5757724103530046, 2.631148267999664, 2.538327346218921, 2.4878734713248507, 2.6133797275761066, 2.6282561527857395, 2.6150327104952447, 3.102757164382848, 3.3318503012160905, 3.3907776288198193, 3.6065313558941936, 3.601180295875859, 3.560491539319038, 3.650095006265445, 3.574812155815713, 3.686227315374108, 3.6338261415040867, 3.5661194785086288, 3.5747332336054645, 3.560674343726918, 3.5678550481603635, 3.5342848534390967, 3.4929538312485913, 3.564544653619436, 3.6861775399566126, 3.6390300636595216, 3.6656336332413666, 3.5731185631923945, 3.5965520044069854, 3.537434489989021, 3.5590937423870144, 3.5331656424410083, 3.640652819618705, 3.5971240740252126, 3.641793843012055, 3.6064014089254295, 3.530378938786505, 3.613631139461306, 3.519542268056021, 3.5416251524576, 3.524789618934195, 3.5519951806099512, 3.6435695455293975, 3.6825670484650863, 3.5993379768209217, 3.628367553897596, 3.633290480934276, 3.5772841681579535, 3.602326323397947, 3.518180278272883, 3.531054006706696, 3.5566645495066167, 3.5410992153240985, 3.630762839301216, 3.5924649123201053, 3.646230633817883, 3.568290612034935, 3.638356129262967, 3.566083243271712, 3.6064978645771797, 3.4942864293427633, 3.595438454812999, 3.681726879126678, 3.6501308156903463, 3.5490717955938593, 3.598535359345363, 3.6328331698421654, 3.595159538698094, 3.556715819008055, 3.6292942886764554, 3.6362895697392856, 3.5965220100874093, 3.6103542985016266, 3.5715010140382493, 3.658769915445062, 3.5939686395400416, 3.4974461928859917, 3.5232691556732267, 3.6145687814416614, 3.5682054018341005, 3.648937250575395, 3.4912089018613384, 3.522426560340423, 3.6757968409374637, 3.651348691084845, 3.5395070091675973, 3.5306275536360383, 3.6153498246329883, 3.599762785949876, 3.5351931286962333, 3.6488316987683054, 3.5198301490992963, 3.5696570079786687, 3.561553836008927, 3.5659475947331423, 3.553147100256108, 3.5475591872743664, 3.6097226797553317, 3.6849600324757934, 3.5264731043844413, 3.506658609738451, 3.5535775980874114, 3.5487291053913554, 3.570651383823912, 3.552993371839188, 3.5054297764661846, 3.5723024888238792]

Here's a monotone curve fitter in essentially 5 lines of python, with numpy
and a lowpass filter from scipy.signal:
#!/usr/bin/env python
"""https://stackoverflow.com/questions/56551114/fully-monotone-interpolation-in-python """
# see also
# https://en.wikipedia.org/wiki/Monotone-spline aka I-spline
# https://scikit-learn.org/stable/modules/isotonic.html
# denis 2 March 2020
from __future__ import division, print_function
import numpy as np
from scipy import signal as sig
from matplotlib import pyplot as plt
import seaborn as sns
def butter_filtfilt( x, Wn=0.5, axis=0 ):
""" butter( 2, Wn ), filtfilt
axis 0 each col, -1 each row
"""
b, a = sig.butter( N=2, Wn=Wn )
return sig.filtfilt( b, a, x, axis=axis, method="gust" ) # twice, forward backward
def ints( x ):
return x.round().astype(int)
def minavmax( x ):
return "min av max %.3g %.3g %.3g" % (
x.min(), x.mean(), x.max() )
def pvec( x ):
n = len(x) // 25 * 25
return "%s \n%s \n" % (
minavmax( x ),
ints( x[ - n : ]) .reshape( -1, 25 ))
#...............................................................................
def monofit( y, Wn=0.1 ):
""" monotone-increasing curve fit """
y = np.asarray(y).squeeze()
print( "\n{ monofit: y %d %s Wn %.3g " % (
len(y), minavmax( y ), Wn ))
ygrad = np.gradient( y )
print( "grad y:", pvec( ygrad ))
# lowpass filter --
gradsmooth = butter_filtfilt( ygrad, Wn=Wn )
print( "gradsmooth:", pvec( gradsmooth ))
ge0 = np.fmax( gradsmooth, 0 )
ymono = np.cumsum( ge0 ) # integrate, sensitive to first few
ymono += (y - ymono).mean()
err = y - ymono
print( "y - ymono:", pvec( err ))
errstr = "average |y - monofit|: %.2g" % np.abs( err ).mean()
print( errstr )
print( "} \n" )
return ymono, err, errstr
#...............................................................................
if __name__ == "__main__":
import sys
np.set_printoptions( threshold=20, edgeitems=15, linewidth=120,
formatter = dict( float = lambda x: "%.2g" % x )) # float arrays %.2g
print( 80 * "=" )
thispy = sys.argv[0]
infile = sys.argv[1] if len(sys.argv) > 1 \
else "so-mono.txt"
Wn = 0.1 # ?
params = "%s %s Wn %g " % (thispy, infile, Wn)
print( params )
y = np.loadtxt( infile ) * 100
print( "y:", y )
ymono, err, errstr = monofit( y, Wn=Wn )
if 1:
sns.set_style("whitegrid")
fig, ax = plt.subplots( figsize=[10, 5] )
plt.subplots_adjust( left=.05, right=.99, bottom=.05, top=.90 )
fig.suptitle(
"Easy monotone curve fit: np.gradient | lowpass filter | clip < 0 | integrate \n"
+ errstr, multialignment="left" )
ax.plot( ymono, color="orangered" )
j = np.where( ymono < y )[0]
xax = np.arange( len(y) )
plt.vlines( xax[j], ymono[j], y[j], color="blue", lw=1 )
j = np.where( ymono > y )[0]
plt.vlines( xax[j], y[j], ymono[j], color="blue", lw=1 )
png = thispy.replace( ".py", ".png" )
print( "writing", png )
plt.savefig( png )
plt.show()

You can try to do preprocessing, to make you input data monotone say using isotonic regression as described at
https://stats.stackexchange.com/questions/156327/fit-monotone-polynomial-to-data

Python OGR: Finding bad spot in source code for code displacement

I've tried to adapt a python ogr script which displaces points too near to each other (originally from http://wiki.gis-lab.info/w/Displacement_of_points_with_same_coordinates,_ShiftPoints/OGR for python2) to Python3. I've also changed the GDAL driver from Shapefile to GeoJSON:
#!/usr/bin/env python
import sys
import math
from itertools import cycle
from optparse import OptionParser
try:
from osgeo import ogr
except ImportError:
import ogr
driverName = "GeoJSON"
class progressBar( object ):
'''Class to display progress bar
'''
def __init__( self, maximum, barLength ):
'''Init progressbar instance.
#param maximum maximum progress value
#param barLength length of the bar in characters
'''
self.maxValue = maximum
self.barLength = barLength
self.spin = cycle(r'-\|/').__next__
self.lastLength = 0
self.tmpl = '%-' + str( barLength ) + 's ] %c %5.1f%%'
sys.stdout.write( '[ ' )
sys.stdout.flush()
def update( self, value ):
'''Update progressbar.
#param value Input new progress value
'''
# Remove last state.
sys.stdout.write( '\b' * self.lastLength )
percent = value * 100.0 / self.maxValue
# Generate new state
width = int( percent / 100.0 * self.barLength )
output = self.tmpl % ( '-' * width, self.spin(), percent )
# Show the new state and store its length.
sys.stdout.write( output )
sys.stdout.flush()
self.lastLength = len( output )
def __log( msg, abort = True, exitCode = 1 ):
''' display log message and abort execution
#param msg message to display
#param abort abort program execution or no. Default: True
#param exitCode exit code passed to sys.exit. Default: 1
'''
print(msg)
if abort:
sys.exit( exitCode )
def doDisplacement( src, dst, radius, rotate, logFile = None, logField = None ):
''' read source file and write displacement result to output file
#param src source shapefile
#param dst destination shapefile
#param radius displacement radius
#param rotate flag indicates rotate points or no
'''
# try to open source shapefile
inShape = ogr.Open( src, False )
if inShape is None:
__log( "Unable to open source shapefile " + src )
print(inShape)
inLayer = inShape.GetLayer( 0 )
print(inLayer)
print(inLayer.GetGeomType())
if inLayer.GetGeomType() not in [ ogr.wkbPoint, ogr.wkbPoint25D ]:
__log( "Input layer should be point layer." )
print(inLayer.GetFeatureCount())
if inLayer.GetFeatureCount() == 0:
__log( "There are no points in given shapefile." )
drv = ogr.GetDriverByName( driverName )
if drv is None:
__log( "%s driver not available." % driverName )
crs = inLayer.GetSpatialRef()
# initialize output shapefile
outShape = drv.CreateDataSource( dst )
if outShape is None:
__log( "Creation of output file %s failed." % dst )
outLayer = outShape.CreateLayer( "point", crs, ogr.wkbPoint )
if outLayer is None:
__log( "Layer creation failed." )
# create fields in output layer
logIndex = 0
featDef = inLayer.GetLayerDefn()
for i in range( featDef.GetFieldCount() ):
fieldDef = featDef.GetFieldDefn( i )
if logField and logField.lower() == fieldDef.GetNameRHef().lower():
logIndex = i
if outLayer.CreateField ( fieldDef ) != 0:
__log( "Creating %s field failed." % fieldDef.GetNameRef() )
__log( "Search for overlapped features", abort = False )
cn = 0
pb = progressBar( inLayer.GetFeatureCount(), 65 )
# find overlapped points
displacement = dict()
inLayer.ResetReading()
inFeat = inLayer.GetNextFeature()
while inFeat is not None:
wkt = inFeat.GetGeometryRef().ExportToWkt()
print(wkt)
if wkt not in displacement:
lst = [ inFeat.GetFID() ]
displacement[ wkt ] = lst
else:
lst = displacement[ wkt ]
j = [ inFeat.GetFID() ]
j.extend( lst )
displacement[ wkt ] = j
cn += 1
pb.update( cn )
inFeat = inLayer.GetNextFeature()
__log( "\nDisplace overlapped features", abort = False )
cn = 0
pb = progressBar( len( displacement ), 65 )
lineIds = []
lineValues = []
lines = []
pc = 0
# move overlapped features and write them to output file
for k, v in displacement.items():
featNum = len( v )
if featNum == 1:
f = inLayer.GetFeature( v[ 0 ] )
if outLayer.CreateFeature( f ) != 0:
__log( "Failed to create feature in output shapefile." )
else:
pc += featNum
fullPerimeter = 2 * math.pi
angleStep = fullPerimeter / featNum
if featNum == 2 and rotate:
currentAngle = math.pi / 2
else:
currentAngle = 0
for i in v:
sinusCurrentAngle = math.sin( currentAngle )
cosinusCurrentAngle = math.cos( currentAngle )
dx = radius * sinusCurrentAngle
dy = radius * cosinusCurrentAngle
ft = inLayer.GetFeature( i )
lineIds.append( str( ft.GetFID() ) )
lineValues.append( ft.GetFieldAsString( logIndex ) )
geom = ft.GetGeometryRef()
x = geom.GetX()
y = geom.GetY()
pt = ogr.Geometry( ogr.wkbPoint )
pt.SetPoint_2D( 0, x + dx, y + dy )
f = ogr.Feature( outLayer.GetLayerDefn() )
f.SetFrom( ft )
f.SetGeometry( pt )
if outLayer.CreateFeature( f ) != 0:
__log( "Failed to create feature in output shapefile." )
f.Destroy()
currentAngle += angleStep
# store logged info
lines.append( ";".join( lineIds ) + "(" + ";".join( lineValues ) + ")\n" )
lineIds = []
lineValues = []
cn += 1
pb.update( cn )
# cleanup
print("\nPoints displaced: %d\n" % pc)
inShape = None
outShape = None
# write log
if logFile:
fl = open( logFile, "w" )
fl.writelines( lines )
fl.close()
if __name__ == '__main__':
parser = OptionParser( usage = "displacement.py [OPTIONS] INPUT OUTPUT",
version = "%prog v0.1",
description = "Shift overlapped points so all of them becomes visible",
epilog = "Experimental version. Use at own risk." )
parser.add_option( "-d", "--distance", action="store", dest="radius",
default=0.015, metavar="DISTANCE", type="float",
help="displacement distanse [default: %default]" )
parser.add_option( "-r", "--rotate", action="store_true", dest="rotate",
default=False, help="place points horizontaly, if count = 2 [default: %default]" )
parser.add_option( "-l", "--log", action="store", dest="logFile",
metavar="FILE",
help="log overlapped points to file [default: %default]. Makes no sense without '-f' option" )
parser.add_option( "-f", "--field", action="store", dest="logField",
metavar="FIELD",
help="Shape field that will be logged to file. Makes no sense without '-l' option" )
( opts, args ) = parser.parse_args()
if len( args ) != 2:
parser.error( "incorrect number of arguments" )
if opts.logFile and opts.logField:
doDisplacement( args[ 0 ], args[ 1 ], opts.radius, opts.rotate, opts.logFile, opts.logField )
else:
doDisplacement( args[ 0 ], args[ 1 ], opts.radius, opts.rotate )
The script
finds displacement candidates by drawing a circle around each point
calculates the angle depending on the number of points to displace (e.g. 2 - 0 & 180)
moves the points
I'm using this input file. Executing python displacement.py input.geojson output.geojson -d 100 should displace all points, but there are always 0 points displaced.
May you please help me finding out the cause?

There is not bad spot in this code, it does exactly what it says it does.
It only displaces points which have the exact same coordinates. Your file has different coordinates. I tried it with a shapefile with the same coordinates, and it worked fine.
I guess you'll have to look for another solution.

How to connect NAO Robot to the wifi with ALConnectionManager, without using the webpage

I would like to connect my NAO robot to a new wifi network with ALConnectionManager, and so I won't have to use the webpage. Using a python API is better than clicking on a webpage...

So here's how I found, not totaly smart, but it works fine on the 2.x.x version...
def connect(strSsid, strPassword):
print( "INF: trying to connect to '%s', with some password..." % (strSsid) )
self.cm = naoqi.ALProxy( "ALConnectionManager")
self.cm.scan();
aS = self.cm.services();
bFound = False;
for s in aS:
print( "\nService: %s" % s )
strServiceID = None;
for attr in s:
attrName, attrValue = attr;
if( attrName == "ServiceId" ):
strServiceID = attrValue;
if( attrName == "Name" and attrValue == strSsid ): # ASSUME, serviceID always set before Name
assert( strServiceID != None );
print( "INF: Good found the right wifi, ServiceID: '%s'\n" % strServiceID )
bFound = True;
break;
if( bFound ):
break;
# for each services - end
if( bFound ):
# real connection
aServiceInput = [];
aServiceInput.append( ["ServiceId", strServiceID ] );
aServiceInput.append( ["Passphrase", strPassword ] );
aServiceInput.append( ["PrivateKeyPassphrase", strPassword ] ); # don't know wich one is usefull!
# aServiceInput.append( ["Favorite", True ] ); # not working
print( "DBG: ServiceInput: %s" % aServiceInput )
retVal = self.cm.setServiceInput( aServiceInput );
print( "DBG: setServiceInput return: %s" % str(retVal) )
retVal = self.cm.connect( strServiceID );
print( "DBG: connect return: %s" % str(retVal) )

How to store data like Freebase does?

I admit that this is basically a duplicate question of Use freebase data on local server? but I need more detailed answers than have already been given there
I've fallen absolutely in love with Freebase. What I want now is to essentially create a very simple Freebase clone for storing content that may not belong on Freebase itself but can be described using the Freebase schema. Essentially what I want is a simple and elegant way to store data like Freebase itself does and be able to easily use that data in a Python (CherryPy) web application.
Chapter 2 of the MQL reference guide states:
The database that underlies Metaweb is fundamentally different than the relational databases that you may be familiar with. Relational databases store data in the form of tables, but the Metaweb database stores data as a graph of nodes and relationships between those nodes.
Which I guess means that I should be using either a triplestore or a graph database such as Neo4j? Does anybody here have any experience with using one of those from a Python environment?
(What I've actually tried so far is to create a relational database schema which would be able to easily store Freebase topics, but I'm having issues with configuring the mappings in SQLAlchemy).
Things I'm looking into
http://gen5.info/q/2009/02/25/putting-freebase-in-a-star-schema/
http://librdf.org/
UPDATE [28/12/2011]:
I found an article on the Freebase blog that describes the proprietary tuple store / database Freebase themselves use (graphd): http://blog.freebase.com/2008/04/09/a-brief-tour-of-graphd/

This is what worked for me. It allows you to load all of a Freebase dump in a standard MySQL installation on less than 100GB of disk. The key is understanding the data layout in a dump and then transforming it (optimizing it for space and speed).
Freebase notions you should understand before you attempt to use this (all taken from the documentation):
Topic - anything of type '/common/topic', pay attention to the different types of ids you may encounter in Freebase - 'id', 'mid', 'guid', 'webid', etc.
Domain
Type - 'is a' relationship
Properties - 'has a' relationship
Schema
Namespace
Key - human readable in the '/en' namespace
Some other important Freebase specifics:
the query editor is your friend
understand the 'source', 'property', 'destination' and 'value' notions described here
everything has a mid, even things like '/', '/m', '/en', '/lang', '/m/0bnqs_5', etc.; Test using the query editor: [{'id':'/','mid':null}]
you don't know what any entity (i.e. row) in the data dump is, you have to get to its types to do that (for instance how do I know '/m/0cwtm' is a human);
every entity has at least one type (but usually many more)
every entity has at least one id/key (but usually many more)
the ontology (i.e. metadata) is embedded in the same dump and the same format as the data (not the case with other distributions like DBPedia, etc.)
the 'destination' column in the dump is the confusing one, it may contain a mid or a key (see how the transforms bellow deal with this)
the domains, types, properties are namespace levels at the same time (whoever came up with this is a genius IMHO);
understand what is a Topic and what is not a Topic (absolutely crucial), for example this entity '/m/03lmb2f' of type '/film/performance' is NOT a Topic (I choose to think of these as what Blank Nodes in RDF are although this may not be philosophically accurate), while '/m/04y78wb' of type '/film/director' (among others) is;
Transforms
(see the Python code at the bottom)
TRANSFORM 1 (from shell, split links from namespaces ignoring notable_for and non /lang/en text):
python parse.py freebase.tsv #end up with freebase_links.tsv and freebase_ns.tsv
TRANSFORM 2 (from Python console, split freebase_ns.tsv on freebase_ns_types.tsv, freebase_ns_props.tsv plus 15 others which we ignore for now)
import e
e.split_external_keys( 'freebase_ns.tsv' )
TRANSFORM 3 (from Python console, convert property and destination to mids)
import e
ns = e.get_namespaced_data( 'freebase_ns_types.tsv' )
e.replace_property_and_destination_with_mid( 'freebase_links.tsv', ns ) #produces freebase_links_pdmids.tsv
e.replace_property_with_mid( 'freebase_ns_props.tsv', ns ) #produces freebase_ns_props_pmids.tsv
TRANSFORM 4 (from MySQL console, load freebase_links_mids.tsv, freebase_ns_props_mids.tsv and freebase_ns_types.tsv in DB):
CREATE TABLE links(
source VARCHAR(20),
property VARCHAR(20),
destination VARCHAR(20),
value VARCHAR(1)
) ENGINE=MyISAM CHARACTER SET utf8;
CREATE TABLE ns(
source VARCHAR(20),
property VARCHAR(20),
destination VARCHAR(40),
value VARCHAR(255)
) ENGINE=MyISAM CHARACTER SET utf8;
CREATE TABLE types(
source VARCHAR(20),
property VARCHAR(40),
destination VARCHAR(40),
value VARCHAR(40)
) ENGINE=MyISAM CHARACTER SET utf8;
LOAD DATA LOCAL INFILE "/data/freebase_links_pdmids.tsv" INTO TABLE links FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n';
LOAD DATA LOCAL INFILE "/data/freebase_ns_props_pmids.tsv" INTO TABLE ns FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n';
LOAD DATA LOCAL INFILE "/data/freebase_ns_base_plus_types.tsv" INTO TABLE types FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n';
CREATE INDEX links_source ON links (source) USING BTREE;
CREATE INDEX ns_source ON ns (source) USING BTREE;
CREATE INDEX ns_value ON ns (value) USING BTREE;
CREATE INDEX types_source ON types (source) USING BTREE;
CREATE INDEX types_destination_value ON types (destination, value) USING BTREE;
Code
Save this as e.py:
import sys
#returns a dict to be used by mid(...), replace_property_and_destination_with_mid(...) bellow
def get_namespaced_data( file_name ):
f = open( file_name )
result = {}
for line in f:
elements = line[:-1].split('\t')
if len( elements ) < 4:
print 'Skip...'
continue
result[(elements[2], elements[3])] = elements[0]
return result
#runs out of memory
def load_links( file_name ):
f = open( file_name )
result = {}
for line in f:
if len( result ) % 1000000 == 0:
print len(result)
elements = line[:-1].split('\t')
src, prop, dest = elements[0], elements[1], elements[2]
if result.get( src, False ):
if result[ src ].get( prop, False ):
result[ src ][ prop ].append( dest )
else:
result[ src ][ prop ] = [dest]
else:
result[ src ] = dict([( prop, [dest] )])
return result
#same as load_links but for the namespaced data
def load_ns( file_name ):
f = open( file_name )
result = {}
for line in f:
if len( result ) % 1000000 == 0:
print len(result)
elements = line[:-1].split('\t')
src, prop, value = elements[0], elements[1], elements[3]
if result.get( src, False ):
if result[ src ].get( prop, False ):
result[ src ][ prop ].append( value )
else:
result[ src ][ prop ] = [value]
else:
result[ src ] = dict([( prop, [value] )])
return result
def links_in_set( file_name ):
f = open( file_name )
result = set()
for line in f:
elements = line[:-1].split('\t')
result.add( elements[0] )
return result
def mid( key, ns ):
if key == '':
return False
elif key == '/':
key = '/boot/root_namespace'
parts = key.split('/')
if len(parts) == 1: #cover the case of something which doesn't start with '/'
print key
return False
if parts[1] == 'm': #already a mid
return key
namespace = '/'.join(parts[:-1])
key = parts[-1]
return ns.get( (namespace, key), False )
def replace_property_and_destination_with_mid( file_name, ns ):
fn = file_name.split('.')[0]
f = open( file_name )
f_out_mids = open(fn+'_pdmids'+'.tsv', 'w')
def convert_to_mid_if_possible( value ):
m = mid( value, ns )
if m: return m
else: return None
counter = 0
for line in f:
elements = line[:-1].split('\t')
md = convert_to_mid_if_possible(elements[1])
dest = convert_to_mid_if_possible(elements[2])
if md and dest:
elements[1] = md
elements[2] = dest
f_out_mids.write( '\t'.join(elements)+'\n' )
else:
counter += 1
print 'Skipped: ' + str( counter )
def replace_property_with_mid( file_name, ns ):
fn = file_name.split('.')[0]
f = open( file_name )
f_out_mids = open(fn+'_pmids'+'.tsv', 'w')
def convert_to_mid_if_possible( value ):
m = mid( value, ns )
if m: return m
else: return None
for line in f:
elements = line[:-1].split('\t')
md = convert_to_mid_if_possible(elements[1])
if md:
elements[1]=md
f_out_mids.write( '\t'.join(elements)+'\n' )
else:
#print 'Skipping ' + elements[1]
pass
#cPickle
#ns=e.get_namespaced_data('freebase_2.tsv')
#import cPickle
#cPickle.dump( ns, open('ttt.dump','wb'), protocol=2 )
#ns=cPickle.load( open('ttt.dump','rb') )
#fn='/m/0'
#n=fn.split('/')[2]
#dir = n[:-1]
def is_mid( value ):
parts = value.split('/')
if len(parts) == 1: #it doesn't start with '/'
return False
if parts[1] == 'm':
return True
return False
def check_if_property_or_destination_are_mid( file_name ):
f = open( file_name )
for line in f:
elements = line[:-1].split('\t')
#if is_mid( elements[1] ) or is_mid( elements[2] ):
if is_mid( elements[1] ):
print line
#
def split_external_keys( file_name ):
fn = file_name.split('.')[0]
f = open( file_name )
f_out_extkeys = open(fn+'_extkeys' + '.tsv', 'w')
f_out_intkeys = open(fn+'_intkeys' + '.tsv', 'w')
f_out_props = open(fn+'_props' + '.tsv', 'w')
f_out_types = open(fn+'_types' + '.tsv', 'w')
f_out_m = open(fn+'_m' + '.tsv', 'w')
f_out_src = open(fn+'_src' + '.tsv', 'w')
f_out_usr = open(fn+'_usr' + '.tsv', 'w')
f_out_base = open(fn+'_base' + '.tsv', 'w')
f_out_blg = open(fn+'_blg' + '.tsv', 'w')
f_out_bus = open(fn+'_bus' + '.tsv', 'w')
f_out_soft = open(fn+'_soft' + '.tsv', 'w')
f_out_uri = open(fn+'_uri' + '.tsv', 'w')
f_out_quot = open(fn+'_quot' + '.tsv', 'w')
f_out_frb = open(fn+'_frb' + '.tsv', 'w')
f_out_tag = open(fn+'_tag' + '.tsv', 'w')
f_out_guid = open(fn+'_guid' + '.tsv', 'w')
f_out_dtwrld = open(fn+'_dtwrld' + '.tsv', 'w')
for line in f:
elements = line[:-1].split('\t')
parts_2 = elements[2].split('/')
if len(parts_2) == 1: #the blank destination elements - '', plus the root domain ones
if elements[1] == '/type/object/key':
f_out_types.write( line )
else:
f_out_props.write( line )
elif elements[2] == '/lang/en':
f_out_props.write( line )
elif (parts_2[1] == 'wikipedia' or parts_2[1] == 'authority') and len( parts_2 ) > 2:
f_out_extkeys.write( line )
elif parts_2[1] == 'm':
f_out_m.write( line )
elif parts_2[1] == 'en':
f_out_intkeys.write( line )
elif parts_2[1] == 'source' and len( parts_2 ) > 2:
f_out_src.write( line )
elif parts_2[1] == 'user':
f_out_usr.write( line )
elif parts_2[1] == 'base' and len( parts_2 ) > 2:
if elements[1] == '/type/object/key':
f_out_types.write( line )
else:
f_out_base.write( line )
elif parts_2[1] == 'biology' and len( parts_2 ) > 2:
f_out_blg.write( line )
elif parts_2[1] == 'business' and len( parts_2 ) > 2:
f_out_bus.write( line )
elif parts_2[1] == 'soft' and len( parts_2 ) > 2:
f_out_soft.write( line )
elif parts_2[1] == 'uri':
f_out_uri.write( line )
elif parts_2[1] == 'quotationsbook' and len( parts_2 ) > 2:
f_out_quot.write( line )
elif parts_2[1] == 'freebase' and len( parts_2 ) > 2:
f_out_frb.write( line )
elif parts_2[1] == 'tag' and len( parts_2 ) > 2:
f_out_tag.write( line )
elif parts_2[1] == 'guid' and len( parts_2 ) > 2:
f_out_guid.write( line )
elif parts_2[1] == 'dataworld' and len( parts_2 ) > 2:
f_out_dtwrld.write( line )
else:
f_out_types.write( line )
Save this as parse.py:
import sys
def parse_freebase_quadruple_tsv_file( file_name ):
fn = file_name.split('.')[0]
f = open( file_name )
f_out_links = open(fn+'_links'+'.tsv', 'w')
f_out_ns = open(fn+'_ns' +'.tsv', 'w')
for line in f:
elements = line[:-1].split('\t')
if len( elements ) < 4:
print 'Skip...'
continue
#print 'Processing ' + str( elements )
#cases described here http://wiki.freebase.com/wiki/Data_dumps
if elements[1].endswith('/notable_for'): #ignore notable_for, it has JSON in it
continue
elif elements[2] and not elements[3]: #case 1, linked
f_out_links.write( line )
elif not (elements[2].startswith('/lang/') and elements[2] != '/lang/en'): #ignore languages other than English
f_out_ns.write( line )
if len(sys.argv[1:]) == 0:
print 'Pass a list of .tsv filenames'
for file_name in sys.argv[1:]:
parse_freebase_quadruple_tsv_file( file_name )
Notes:
Depending on the machine the index creation may take anywhere from a few to 12+ hours (consider the amount of data you are dealing with though).
To be able to traverse the data in both directions you need an index on links.destination as well which I found to be expensive timewise and never finished.
Many other optimizations are possible here. For example the 'types' table is small enough to be loaded in memory in a Python dict (see e.get_namespaced_data( 'freebase_ns_types.tsv' ))
And the standard disclaimer here. It has been a few months since I did this. I believe it is mostly correct but I do apologize if my notes missed something. Unfortunately the project I needed it for fell through the cracks but hope this helps someone else. If something isn't clear drop a comment here.

My 2 cents...
I use a little bit of Java code to convert the Freebase data dump into RDF: https://github.com/castagna/freebase2rdf
I use Apache Jena's TDB store to load the RDF data and Fuseki to serve the data via SPARQL protocol over HTTP.
See also:
http://markmail.org/thread/mq6ylzdes6n7sc5o
http://markmail.org/thread/jegtn6vn7kb62zof

SPARQL is the query language to query RDF, it allows to write SQL-alike queries. Most RDF databases implement SPARQL interfaces. Moreover, Freebase allows you to export data in RDF so you could potentially use that data directly in an RDF database and query it with SPARQL.
I would have a look at this tutorial to get a better sense of SPARQL.
If you are going to handle a big dataset, like freebase, I would use 4store together with any of the Python clients. 4store exposes SPARQL via HTTP, you can make HTTP requests to assert, remove and query data. It also handles resultsets in JSON, and this is really handy with Python. I have used this infrastructure in several projects, not with CherryPy but with Django, but I guess that this difference doesn't really matter.

A good news for freebase dump users is that Freebase now offer RDF dump now: http://wiki.freebase.com/wiki/Data_dumps . It is in turtle format, so it is very convenient to use any graph database designed for RDF.
My suggestion is also 4store: http://4store.org/ . it is simple and easy to use.
You could use http request to do the SPARQL operation.
One tricky thing in my project is that the "." used in Freebase dump (to represent shorten URL) is not recognizable to 4store. So I add a bracket "<>" o all the columns contained "." and deal with the shorten URL myself.

Have a look at https://cayley.io. I believe it is written by the same author and uses same principles as graphd, the backend of Freebase, before Google killed it.
Regarding the data, you probably will want to run something like this to cleanup the Freebase DB dumps or use datahub.

And this is the extra code for my other answer. The meat is in edb.py. Run from Python console and follow the examples. Or use the web2py controller and run in your browser.
Save this as edb.py:
import MySQLdb
import sys
connection = MySQLdb.connect (host = "localhost",
user = "root",
passwd = "x",
db = "y")
cursor = connection.cursor()
query_counter = 0
print_queries = False
limit = 1000
def fetch_one( query ):
global query_counter, print_queries
query = query + ' LIMIT ' + str(limit)
if print_queries:
print query
cursor = connection.cursor()
cursor.execute( query )
query_counter += 1
result = cursor.fetchone()
if result:
return result[0]
else:
return None
def fetch_all( query ):
global query_counter, print_queries
query = query + ' LIMIT ' + str(limit)
if print_queries:
print query
cursor = connection.cursor()
cursor.execute( query )
query_counter += 1
return cursor.fetchall()
def _flatten( list_of_lists ):
import itertools
return list(itertools.chain(*list_of_lists))
#Example: e._search_by_name('steve martin')
def _search_by_name( name, operator = '=' ):
typed, ranked = {}, []
if name:
name = name.strip()
if not name:
return ( typed, ranked )
filler = '' if operator == '=' else '%'
ranks = {}
#to filter meaningful stuff for every mid returned order by the number of types they have
#search for value text if prop. is
#select * from ns where value = 'the king' and (property = '/m/01gr' or property = '/m/06b');
name_mid = _mid( '/type/object/name' )
alias_mid = _mid( '/common/topic/alias' )
query = "select ns.source from ns where ns.value %s '%s%s' and ns.property in ('%s', '%s')" % ( operator, name, filler, name_mid, alias_mid )
for i in fetch_all( query ):
typed[ i[0] ] = _types( i[0] )
import operator
ranked = [ ( len( typed[i] ), i ) for i in typed ]
ranked = [ e[1] for e in sorted( ranked, key=operator.itemgetter(0), reverse = True ) ]
return (typed, ranked)
#Example: e._children('') <---will get the top level domains
# e._children('/film') <---get all types from the domain
# e._children('/film/film') <---get all properties for the type
def _children( parent, expand = False, raw = False ):
query = "select t.source, t.value from types t where t.destination = '%s'" % (parent)
res = fetch_all( query )
if raw:
return [ row[0] for row in res ]
if expand: prefix = parent
else: prefix = ''
return [ prefix + '/' + row[1] for row in fetch_all(query) ]
#Example: e._parent('/film/film/songs')
def _parent( child ): # '/people/marriage/to' -> '/people/marriage'
#if not isinstance( child, str ): return None # what kind of safety mechanisms do we need here?
return '/'.join(child.split('/')[:-1])
#Example: e._domains()
def _domains():
return _children('')
#Example: e._top_level_types()
def _top_level_types():
return _children('/type')
#TODO get all primitive types
#Example: e._mid('/type/object')
# e._mid('/authority/imdb/name/nm0000188')
def _mid( key ):
if key == '':
return None
elif key == '/':
key = '/boot/root_namespace'
parts = key.split('/')
if parts[1] == 'm': #already a mid
return key
namespace = '/'.join(parts[:-1])
key = parts[-1]
return fetch_one( "select source from types t where t.destination = '%s' and t.value = '%s'" % (namespace, key) )
#Example: e._key('/type')
def _key( mid ):
if isinstance( mid, str):
res = _keys( mid )
if not res:
return None
rt = [ r for r in res if r.startswith( '/type' ) ]
if rt:
return rt[0]
else:
return res[0]
elif isinstance( mid, list ) or isinstance( mid, tuple ):
res = [ _key( e ) for e in mid ]
return [ r for r in res if r is not None ]
else:
return None
def _keys( mid ):
# check for '/type/object/key' as well?
query = "select t.destination, t.value from types t where t.source = '%s'" % mid
return [ row[0]+'/'+row[1] for row in fetch_all( query ) ]
#Example: e._types('/m/0p_47')
def _types( mid ):
tm = _mid( '/type/object/type' )
query = "select l.destination from links l where l.source = '%s' and l.property = '%s'" % (mid, tm)
return [ row[0] for row in fetch_all( query ) ]
#Example: e._props_n('/m/0p_47') <---Named immediate properties (like name, etc.)
def _props_n( mid ): #the same property can be set more than once per topic!
query = "select ns.property from ns where ns.source = '%s'" % (mid)
return list( set( [ row[0] for row in fetch_all( query ) ] ) )
#Example: e._props_l('/m/0p_47') <---All remote properties, some are named, some are anonymous
def _props_l( mid ): #the same property can be set more than once per topic!
tm = _mid( '/type/object/type' ) #exclude types, they have tons of instance links
res = fetch_all( "select l.property, l.destination from links l where l.source = '%s' and property <> '%s'" % (mid, tm) )
output = {}
for r in res:
dests = output.get( r[0], False )
if dests:
dests.append( r[1] )
else:
output[ r[0] ] = [ r[1] ]
return output
#Example: e._props_ln('/m/0p_47') <---All remote named properties
def _props_ln( mid ): #named properties
result = []
ps = _props_l( mid )
common_topic = _mid( '/common/topic' )
for p in ps:
ts = _types( ps[p][0] )
if common_topic in ts: #it's a common topic
result.append( p )
return result
#Example: e._props_la('/m/0p_47') <---All remote anonymous properties, these actually belong to the children!
#instead of has type /common/topic we used to check if it has name
def _props_la( mid, raw = True ): #anonymous properties (blank nodes in RDF?)
result = []
ps = _props_l( mid )
common_topic = _mid( '/common/topic' )
for p in ps:
ts = _types( ps[p][0] )
if common_topic not in ts: #it is not a common topic
t = _key( _types( ps[p][0] ) )
if t and '/type/type' not in t: #FIXME: hack not to go into types, could be done better
result.append( _children( t[0], expand=True, raw=raw ) ) #get the first, is this correct?
return _flatten( result ) #it is a list of lists
#FIXME: try to get '/film/actor/film' -> '/type/property/expected_type' -> '/film/performance' -> properties/children
#instead of trying is something has name
#Example: e._get_n('/m/0p_47', e._props_n('/m/0p_47')[0])['/lang/en'] <---These come with a namespace
def _get_n( mid, prop ): #the same property can be set more than once per topic!
p = _mid( prop )
query = "select ns.value from ns where ns.source = '%s' and ns.property = '%s'" % (mid, p)
return [ r[0] for r in fetch_all( query ) ]
#Example: e._get_l('/m/0p_47', e._props_l('/m/0p_47')[0]) <---returns a list of mids coresponding to that prop.
# e._name(e._get_l('/m/0p_47', '/film/writer/film'))
def _get_l( mid, prop ): #the same property can be set more than once per topic!
p = _mid( prop )
query = "select l.destination from links l where l.source = '%s' and l.property = '%s'" % (mid, p)
return [ row[0] for row in fetch_all( query ) ]
#Example: e._name(e._get_ln('/m/0p_47', e._props_ln('/m/0p_47')[0]))
def _get_ln( mid, p ): #just alias for _get_l, keeping for consistency
return _get_l( mid, p )
#Example: e._name(e._get_la('/m/0p_47', '/film/performance/film'))
def _get_la( mid, prop ):
result = []
ps = _props_l( mid )
for p in ps:
es = _get_l( mid, p ) #get the destinations
if not es: continue
ts = set( _types( es[0] ) )
if _mid(_parent(_key(_mid(prop)))) in ts: #should be able to do this more efficiently!!!
for e in es:
result.append( _get_l( e, prop ) )
return _flatten( result ) #return after the first result
#How do we determine properties with multiple values vs those with singular (i.e. place of birth)?
#is this in the ontology?
#Ans: yes, /type/property/unique
#Example: e._all_names_ln('/m/0p_47') <---gets all of object's remote named properties
def _all_names_ln( mid ):
result = {}
for p in _props_ln( mid ):
result[ _key(p) ] = _name( _get_ln( mid, p ) )
return result
#Example: e._all_names_la('/m/0p_47') <---gets all of object's remote anonymous properties
def _all_names_la( mid ): #TODO: prevent loops, run e.all_names_la('/m/0p_47')
result = {}
for p in _props_la( mid ):
result[ _key( p ) ] = _name ( _get_la( mid, p ) )
return result
#FIXME: _all_names_la is going into destinations which are types and have a ton of instance links...
#Example: e._name('/m/0p_47') <---the name of a topic
#
def _name( mid ):
if isinstance( mid, str ):
nm = _mid( '/type/object/name' )
return _get_n( mid, nm )
elif isinstance( mid, list ) or isinstance( mid, tuple ) or isinstance( mid, set ):
return [ _name( e ) for e in mid ]
else:
return None
#for internal use only
def _get_linked( mid ):
tm = _mid( '/type/object/type' ) #exclude types, they have tons of instance links
query = "select destination from links where source = '%s' and property <> '%s' " % ( mid, tm )
return set( [ r[0] for r in fetch_all( query ) ] )
#for internal use only
def _get_connections_internal( entity1, target, path, all_paths, depth, max_depth):
import copy
if depth > max_depth:
return
if True:
print
print str(entity1) + ', ' + str(target)
print str( path )
print str( all_paths )
print depth
path.append( entity1 )
linked1 = _get_linked( entity1 )
if target in linked1 or entity1 == target:
path.append( target )
all_paths.append( path )
#print str( path )
return
for l1 in linked1:
if l1 in path:
continue
_get_connections_internal( l1,
target,
copy.copy( path ),
all_paths,
depth+1,
max_depth )
#Example: e._name(e._get_connections('/m/0p_47', '/m/0cwtm')) <---find path in the graph between the two entities
def _get_connections( entity1, target ):
result = []
_get_connections_internal( entity1, target, [], result, 0, 2 )
return result
#for internal use only
def _get_connections_internal2( entity1, entity2, path1, path2, all_paths, depth, max_depth, level ):
import copy
if depth > max_depth:
return
if level < 0: level = 0
path1.append( entity1 )
path2.append( entity2 )
if entity1 == entity2 and level == 0:
all_paths.append( ( path1, path2 ) ) #no need to append entity1 or entity2 to the paths
return
linked1 = _get_linked( entity1 )
if entity2 in linked1 and entity2 not in path1 and level == 0:
path1.append( entity2 )
all_paths.append( ( path1, path2 ) )
return
linked2 = _get_linked( entity2 )
if entity1 in linked2 and entity1 not in path2 and level == 0:
path2.append( entity1 )
all_paths.append( ( path1, path2 ) )
return
inters = linked1.intersection( linked2 )
inters = inters.difference( set( path1 ) )
inters = inters.difference( set( path2 ) )
if inters and level == 0:
for e in inters: #these are many paths, have to clone
p1 = copy.copy( path1 )
p1.append( e )
p2 = copy.copy( path2 )
p2.append( e )
all_paths.append( ( p1,p2 ) )
return
for l1 in linked1:
if l1 in path1 or l1 in path2:
continue
for l2 in linked2:
if l2 in path1 or l2 in path2:
continue
_get_connections_internal2( l1, l2,
copy.copy( path1 ), copy.copy( path2 ),
all_paths,
depth+1,
max_depth,
level - 1 )
#Example: e._name(e._get_connections2('/m/0p_47', '/m/0cwtm')) <---returns two meeting paths starting from both entities
# e._name(e._get_connections('/m/0p_47', '/m/0cwtm', level=1)) <---search deeper
# e._name(e._get_connections('/m/0p_47', '/m/0cwtm', level=2)) <---even deeper
def _get_connections2( entity1, entity2, level = 0 ):
result = []
_get_connections_internal2( entity1, entity2, [], [], result, 0, 15, level )
return result
And here is a sample web2py controller (just copy edb.py in the web2py models directory):
# -*- coding: utf-8 -*-
def mid_to_url( mid ):
return mid.split('/')[2]
def index():
form = FORM( TABLE( TR( INPUT(_name='term', _value=request.vars.term ) ),
TR(INPUT(_type='submit', _value='Search') ) ),
_method='get')
typed, ranked = _search_by_name( request.vars.term )
rows = []
for r in ranked:
keys = []
for t in typed[r]:
k = _key( t )
if k:
keys.append( k )
rows.append( TR( TD( A(_name( r ),
_href = URL('result', args = [mid_to_url(r)]))),
TD( XML( '<br/>'.join( keys ) ) ) ) )
result = TABLE( *rows )
return {
'form': form,
'result' : result
}
def result():
path, data = '', ''
if not request.args:
return { 'path':path, 'data':data}
path_rows = []
for ra in range(len(request.args)):
if ra%2:
arrow_url = URL( 'static', 'images/blue_arr.png' )
display_name = _key('/m/'+request.args[ra]) #it's a property
else:
arrow_url = URL( 'static', 'images/red_arr.png' )
display_name = _name('/m/'+request.args[ra]) #it's a topic
path_rows.append( TD( A( display_name, _href=URL( args = request.args[0:ra+1] ) ) ) )
path_rows.append( TD( IMG( _src = arrow_url ) ) )
path = TABLE( *path_rows )
elems = [ '/m/'+a for a in request.args ]
if _mid( '/type/property' ) in _types( elems[-1] ): #we are rendering a property
objects = _get_ln( elems[-2], elems[-1] )
if not objects: #there should be a better way to see if this is anonymous
objects = _get_la( elems[-2], elems[-1] )
data = TABLE( *[ TR( TD( A(_name(o), _href = URL( args = request.args+[mid_to_url(o)])))) for o in objects ] )
else: #we are rendering a topic
direct_props = TABLE(*[TR(TD(_key(p)), TD(', '.join(_get_n( elems[-1], p)))) for p in _props_n( elems[-1] )])
linked_named_props = TABLE(*[TR(TD(A(_key(p),
_href = URL(args = request.args+[mid_to_url(p)])))) for p in _props_ln( elems[-1] ) ] )
linked_anon_props = TABLE(*[TR(TD(A(_key(p),
_href = URL(args = request.args+[mid_to_url(p)])))) for p in _props_la( elems[-1] ) ] )
data = TABLE( TR( TH( 'Linked named data:'), TH( 'Linked anonymous data:' ), TH( 'Direct data:' ) ),
TR( TD( linked_named_props ), TD( linked_anon_props ), TD( direct_props ) ) )
return { 'path': path, 'data':data }

Add XML to a Text Box

I want to add an xml to a QTextEdit, this is my code
self.XMLField = QtGui.QTextEdit() # Alternative: QTextEdit
self.XMLField.setReadOnly( True )
self.XMLField.setAcceptRichText( True )
self.XMLField.append( data.toxml() )
print( data.toxml() )
The print is working, so I get the whole XML, but in the text box I only get the nodeValues and attributes.
ADDED
This is the full code:
path = "settings/%s.xml" % str( self.clientName )
print( path );
data = xml.dom.minidom.parse( path )
lidar = data.getElementsByTagName( 'lidar' )
if( lidar.length > 0 ):
positive_towards_LOS = lidar[0].getAttribute( 'positive_towards_LOS' )
scanner_3D = lidar[0].getAttribute( 'scanner_3D' )
name = ( lidar[0].getElementsByTagName( 'name' ) )
if( name.length > 0 ):
title = 'Windscanner - %s Lidar Properties' % name[0].firstChild.nodeValue
self.setWindowTitle( title )
""" XML Showup """
self.XMLField = QtGui.QTextEdit() # Alternative: QTextEdit
self.XMLField.setReadOnly( True )
self.XMLField.setAcceptRichText( True )
self.XMLField.append( data.toxml() )

We Keep Coding

Python is a programming language that lets you work quickly and integrate systems more effectively.