I am currently trying to generate a heatmap in python from a text file, using R commands (with rpy2). It works fine in R, but when I take it to python, the Quartz interface displays quickly and then closes. I would like either to be able to save the quartz display to a file, or directly save my heatmap to a file without displaying it.
Here is the code I have been using:
import rpy2.robjects as robjects
robjects.r('''
library("gplots")
data = read.csv("/Users/.../Heatmap_data.txt")
DF = data.frame(data)
MD = data.matrix(DF,rownames.force=NA)
heatmap.2(MD, scale="none", col=redgreen(100), cexRow=0.1, key=FALSE, symkey=FALSE, trace="none", Colv=FALSE)
''')
I'm using python 2.7, on OS X Yosemite.
Thank you for any help.
import numpy as np
import rpy2.robjects as ro
import rpy2.robjects.numpy2ri
ro.numpy2ri.activate()
R = ro.r
data = np.random.random((10, 10))
R.png(file='/tmp/out.png')
R.heatmap(data)
R("dev.off()")
writes to the file /tmp/out.png without displaying the image:
.
Preventing the displayed image from immediately closing can be done like this:
script.py:
import numpy as np
import rpy2.robjects as ro
import rpy2.robjects.numpy2ri
import rpy2.rinterface as rinterface
import time
import threading
ro.numpy2ri.activate()
R = ro.r
def ion():
def r_refresh(interval = 0.03):
while True:
rinterface.process_revents()
time.sleep(interval)
t = threading.Thread(target=r_refresh)
t.daemon = True
t.start()
ion()
data = np.random.random((10, 10))
R.heatmap(data)
R("dev.copy(png,'/tmp/out2.png')")
R("dev.off()")
try:
# for Python2
raw_input()
except NameError:
# for Python3
input()
The raw_input or input call prevents the Python interpreter from exiting, thus allowing the window to stay open, until the user presses Enter.
The ion function calls rinterface.process_revents() periodically so the
displayed window will react to GUI events such as resizing or being closed.
dev.copy(png,'/tmp/out2.png') saves the already-displayed image to a
file.
Related
I am trying to run a python file that works fine on my Windows machine on a remote server:
import collections
import pandas as pd
from pathlib import Path
import shelve
import cloudpickle
import numpy as np
import typing
from typing import List
import ray
from ray.rllib.agents import ppo
from ray.rllib.utils.spaces.space_utils import flatten_to_single_ndarray
from ray.tune import register_env
from ray.rllib.env.base_env import _DUMMY_AGENT_ID
from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID
.
.
.
if __name__ == '__main__':
WIN = False
ray.init(dashboard_port=8263)
daterange = pd.date_range('2017-01-01', periods=35040, freq='15T')
norming_factor = 10
actions_module = ActionsModuleContinuous()
batch_n_days = 1
kappa = 1000
seed = 1234
steps_per_episode = batch_n_days * 24 * 4
num_episodes = 5
device_config: List = list()
device_config.append(Device1(WIN=WIN, norming_factor=norming_factor, n_data_points=steps_per_episode))
device_config.append(Device2(WIN=WIN))
device_config.append(Device3(WIN=WIN))
It returns the following SyntaxError, for no apparent reason:
(venv) [<username>#<server> examples]$ python test.py
File "test.py", line 183
device_config: List = list()
^
SyntaxError: invalid syntax
I have tried deleting : List, which just had the effect of moving the same error to a seemingly arbitrary place further down the script. Any help is greatly appreciated.
device_config: List = list()
That syntax is adding type annotations for variable device_config.
If you get SyntaxError: invalid syntax exception, it means that your python interpreter used to run your code is not new enough. Nothing to do with your code, it just means that when you enter python in your shell, interpreter that gets executed is not new enough.
I would like to read a file in a remote machine. I can do it using paramiko.
The file is constantly updated by newlines. I have tried to implement a python script for reading it. Here the interesting part of the code:
import glob
import sys
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import os
import pandas as pd
from scipy.linalg import norm
import time
import paramiko
import select
if __name__ == "__main__":
print("...starting")
# a lot of stuff here in the middle
ssh_client = paramiko.SSHClient()
ssh_client.load_system_host_keys()
ssh_client.connect(hostname='xxx.xx.xx.xxx',username='user',password='pass')
print("...starting transport:")
transport = ssh_client.get_transport()
channel = transport.open_session()
channel.exec_command("cat /tmp/ciao.txt")
while True:
rl, wl, xl = select.select([channel],[],[],0.0)
#print(rl.readlines())
if len(rl) > 0:
#print("printing")
string_in_file = channel.recv(1024)
if len(string_in_file) > 0:
#print("printing")
print(string_in_file)
Problem: the file is correctly read at the beginning and after, every newly written line is completely ignored or, at least, it does not produce any effect on the output of the proposed script. Any suggestions on how to read new lines when written?
Any other idea on how to achieve the same result (even without paramiko) is more than welcome. The only restriction is the use of python.
tail -f will keep following the file giving you more output as you go.
import glob
import sys
import os
import time
import paramiko
import select
if __name__ == "__main__":
print("...starting")
# a lot of stuff here in the middle
ssh_client = paramiko.SSHClient()
ssh_client.load_system_host_keys()
# for test get "user,pw" in ./test.pw
user, pw = open('test.pw').readline().strip().split(",")
ssh_client.connect(hostname='localhost',username=user,password=pw)
print("...starting transport:")
transport = ssh_client.get_transport()
channel = transport.open_session()
# 1GB is include first Gig - just a way to get all of the
# file instead of the last few lines
# include --follow=name instead of -f if you want to keep
# following files that are log rotated
channel.exec_command("tail -f --lines=1GB /tmp/test.txt")
while True:
# (don't melt cpu's with a zero timeout)
rl, wl, xl = select.select([channel],[],[])
#rl, wl, xl = select.select([channel],[],[],0.0)
if rl:
string_in_file = channel.recv(1024)
if len(string_in_file) > 0:
print(string_in_file)
else:
print("channel disconnected")
break
I want to work with the APIs of the program of structural analysis (civil engineering) Autodesk Robot Structural Analysis.
With IronPython I initialize the variables as follows:
import clr
clr.AddReferenceToFileAndPath(‘mypfad\Interop.RobotOM.dll’)
from RobotOM import *
robapp = RobotApplicationClass()
robproj = robapp.Project
robstruct = robproj.Structure
With robstruct I can call the API functions and continue working.
Now I’d like to do the same with Python 3. I have tried with ctypes and with numpy.ctypeslib without success:
import ctypes
lib_ctypes = ctypes.cdll[‘mypfad\Interop.RobotOM.dll']
print(lib_ctypes)
<CDLL 'mypfad \Interop.RobotOM.dll', handle 1a1ff900000 at 0x1a1e8e22710>
import numpy
lib_numpy = numpy.ctypeslib.load_library('Interop.RobotOM.dll', 'mypfad’)
print(lib_ numpy)
<CDLL 'mypfad\Interop.RobotOM.dll', handle 1a1ffb40000 at 0x1a1ffb194e0>
And I don’t how to continue.
My questions are: is this the right way and how shall I continue?
Edited 05.10.2018
Original code with IronPython:
import clr
# Add Robot Structural Analysis API reference
clr.AddReferenceToFileAndPath(
'C:\Program Files\Common Files\Autodesk Shared\Extensions 2018\Framework\Interop\Interop.RobotOM.dll'
)
# Add needed import to be able to use Robot Structural Analysis objects
from RobotOM import *
# Connect to the running instance of Robot Structural Analysis
robapp = RobotApplicationClass()
# Get a reference of the current project
robproj = robapp.Project
# Get a reference of the current model
robstruct = robproj.Structure
An attempt according to the comment of The Machine:
import ctypes
my_dll = ctypes.cdll.LoadLibrary(
'C:\Program Files\Common Files\Autodesk Shared\Extensions 2018\Framework\Interop\Interop.RobotOM.dll'
)
robapp = my_dll.RobotApplicationClass()
robproj = robapp.Project
robstruct = robproj.Structure
Result:
AttributeError: function 'RobotApplicationClass' not found
Edited 16.10.2018
Third attempt:
from ctypes import cdll
my_dll = cdll.LoadLibrary(
'C:\Program Files\Common Files\Autodesk Shared\Extensions 2019\Framework\Interop\Interop.RobotOM.dll'
)
my_dll.RobotApplicationClass()
Result:
AttributeError: function 'RobotApplicationClass' not found
Try this:
from ctypes import*
your_dll = cdll.LoadLibrary(‘mypfad\Interop.RobotOM.dll ')
If it is succesfully loaded than you can aces to all of classes and function in that dll file . You can call theme with your_dll.nameofclass .
My specific issue is exactly the title. I have a large raster processing script in python and need to perform a clump function which I cannot find in gdal / python nor have I figured out how to 'write it' myself.
I am becoming better with python all the time just still newish, but am learning R for this task. (installed R version 3.4.1 (2017-06-30))
I am able to get rpy2 installed within python after spending a little time learning R and through help on Stackoverflow I have been able to perform several 'tests' of rpy2.
The most helpful info in getting rpy2 to respond was to establish where your R is within your python session or script. from another Stack answer. As below:
import os
os.environ['PYTHONHOME'] = r'C:\Python27\ArcGIS10.3\Scripts\new_ve_folder\Scripts'
os.environ['PYTHONPATH'] = r'C:\Python27\ArcGIS10.3\Scripts\new_ve_folder\Lib\site-packages'
os.environ['R_HOME'] = r'C:\Program Files\R\R-3.4.1'
os.environ['R_USER'] = r'C:\Python27\ArcGIS10.3\Scripts\new_ve_folder\Lib\site-packages\rpy2'
However, the main tests listed in the documentation http://rpy.sourceforge.net/rpy2/doc-2.1/html/overview.html I cannot get to work.
import rpy2.robjects.tests
import unittest
# the verbosity level can be increased if needed
tr = unittest.TextTestRunner(verbosity = 1)
suite = rpy2.robjects.tests.suite()
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
AttributeError: 'module' object has no attribute 'suite'
However:
import rpy2.robjects as robjects
pi = robjects.r['pi']
pi[0]
works just fine. as do a few other rpy2.robjects tests I have found. I can create string = ''' f <- functions ect ''' and call those from python.
If i use:
python -m 'rpy2.tests'
I get the following error.
r\Scripts>python -m 'rpy2.tests'
r\Scripts\python.exe: No module named 'rpy2
Documentation states: On Python 2.6, this should return that all tests were successful. I am using Python 2.7 and I also tried this in Python 3.3.
My script for clump starts as below:
I do not want to have to actually install the package names each time I run the script as they are already installed in my R Home.
I would like to use my python variables if possible.
I need to figure out why rpy2 does not respond as the documentation indicates, or why I am getting errors. And then after that figure out the correct way to write my clump portion of my python script.
packageNames = ('raster', 'rgdal')
if all(rpackages.isinstalled(x) for x in packageNames):
have_packages = True
else:
have_packages = False
if not have_packages:
utils = rpackages.importr('utils')
utils.chooseCRANmirror(ind=1)
packnames_to_install = [x for x in packageNames if not rpackages.isinstalled(x)]
if len(packnames_to_install) > 0:
utils.install_packages(StrVector(packnames_to_install))
from rpy2.robjects.packages import importr
import rpy2.robjects as robjects
There are several ways I have found to call the raster and clump options from R, however, if I cannot get rpy2 to respond correctly, I am not going to get these to work at all But since several other tests work I am not positive.
raster = robjects.r['raster']
raster = importr('raster')
clump = raster.clump
clump = robjects.r.clump
type(raster.clump)
tempDIR = r"C:\Users\script_out\temp"
slope_recode = os.path.join(tempDIR, "step2b_input.img")
outfile = os.path.join(tempDIR, "Rclumpfile.img")
raster.clump(slope_recode, filename=outfile, direction=4, gaps=True, format='HFA', overwrite=True)
Which results in a large amount of errors.
Traceback (most recent call last):
File "C:/Python27/ArcGIS10.3/Scripts/new_ve_folder/Scripts/rpy2_practice.py", line 97, in <module>
raster.clump(slope_recode, filename=outfile, direction=4, gaps=True, format='HFA', overwrite=True)
File "C:\Python27\ArcGIS10.3\Scripts\new_ve_folder\lib\site-packages\rpy2\robjects\functions.py", line 178, in __call__
return super(SignatureTranslatedFunction, self).__call__(*args, **kwargs)
File "C:\Python27\ArcGIS10.3\Scripts\new_ve_folder\lib\site-packages\rpy2\robjects\functions.py", line 106, in __call__
res = super(Function, self).__call__(*new_args, **new_kwargs)
rpy2.rinterface.RRuntimeError: Error in (function (classes, fdef, mtable) :
unable to find an inherited method for function 'clump' for signature '"character"'
Issues:
testing rpy2 in command line and script (both produce errors, but I am still able to use basic rpy2
importing the R packages so as not to install them each time
finally getting my clump script called correctly
If I have missed something basic, please point me in the right direction. Thanks all.
For your first problem, replace suite = rpy2.robjects.tests.suite() with suite = rpy2.tests.suite().
For your third problem (getting clump to work correctly), you need to create a RasterLayer object in R using the image. I'm not familiar with the raster package, so I can't give you the exact steps.
I will point out the arcpy module is not "pythonic". Normally, strings of filenames are just strings in Python. arcpy is weird in using plain strings to represent objects like map layers.
In your example, slope_recode is just a string. That's why you got the error unable to find an inherited method for function 'clump' for signature '"character"'. It means slope_recode was passed to R as a character value (which it is), and the clump function expects a RasterLayer object. It doesn't know how to handle character values.
I got this all to work with the below code.
import warnings
os.environ['PATH'] = os.path.join(scriptPath, 'path\\my_VE\\R\\R-3.4.2\\bin\\x64')
os.environ['PYTHONHOME'] = os.path.join(scriptPath, 'path\\my_VE\\Scripts\\64bit')
os.environ['PYTHONPATH'] = os.path.join(scriptPath, 'path\\my_VE\\Lib\\site-packages')
os.environ['R_HOME'] = os.path.join(scriptPath, 'path\\my_VE\\R\\R-3.4.2')
os.environ['R_USER'] = os.path.join(scriptPath, 'path\\my_VE\\Scripts\\new_ve_folder\\Scripts\\rpy2')
#
import platform
z = platform.architecture()
print(z)
## above will confirm you are working on 64 bit
gc.collect()
## this code snippit will tell you which library is being Read
command = 'Rscript'
cmd = [command, '-e', ".libPaths()"]
print(cmd)
x = subprocess.Popen(cmd, shell=True)
x.wait()
import rpy2.robjects.packages as rpackages
import rpy2.robjects as robjects
from rpy2.robjects import r
import rpy2.interactive.packages
from rpy2.robjects import lib
from rpy2.robjects.lib import grid
# # grab r packages
print("loading packages from R")
## fails at this point with the following error
## Error: cannot allocate vector of size 232.6 Mb when working with large rasters
rpy2.robjects.packages.importr('raster')
rpy2.robjects.packages.importr('rgdal')
rpy2.robjects.packages.importr('sp')
rpy2.robjects.packages.importr('utils')
# rpy2.robjects.packages.importr('memory')
# rpy2.robjects.packages.importr('dplyr')
rpy2.robjects.packages.importr('data.table')
grid.activate()
# set python variables for R code names
raster = robjects.r['raster']
writeRaster = robjects.r['writeRaster']
# setwd = robjects.r['setwd']
clump = robjects.r['clump']
# head = robjects.r['head']
crs = robjects.r['crs']
dim = robjects.r['dim']
projInfo = robjects.r['projInfo']
slope_recode = os.path.join(tempDIR, "_lope_recode.img")
outfile = os.path.join(tempDIR, "Rclumpfile.img")
recode = raster(slope_recode) # this is taking the image and reading it into R raster package
## https://stackoverflow.com/questions/47399682/clear-r-memory-using-rpy2
gc.collect() # No noticeable effect on memory usage
time.sleep(2)
gc.collect() # Finally, memory usage drops
R = robjects.r
R('memory.limit()')
R('memory.limit(size = 65535)')
R('memory.limit()')
print"starting Clump with rpy2"
clump(recode, filename=outfile, direction=4, gaps="True", format="HFA")
final = raster(outfile)
final = crs("+proj=longlat +datum=WGS84 +ellps=WGS84 +towgs84=0,0,0,-0,-0,-0,0 +no_defs")
print ("clump file created, CRS accurate, next step")
I'm currently using parallel python ,and in the parameters of job_server.submit i added the library in modules but the problem is that even that library has other librairies in it .so what should i do ?
Here is the code i'm trying to run:
from tools.demo import detect_cn
import pp
job_server = pp.Server()
f1 = job_server.submit(detect_cn, (filename,),modules=('tools.demo',))
f2 = job_server.submit(detect_cn, (filename1,),modules=('tools.demo',))
cnis, preproc_time, roi_file_images=f1()
cnis1, preproc_time1, roi_file_images1=f2()
and this is part of code of demo.py
import _init_paths
from fast_rcnn.config import cfg
from fast_rcnn.test import im_detect
from fast_rcnn.nms_wrapper import nms
from utils.timer import Timer
from ocr.clstm import clstm_ocr
from ocr.clstm import clstm_ocr_calib
import matplotlib.pyplot as plt
import numpy as np
import scipy.io as sio
import caffe, os, sys, cv2
import argparse
import werkzeug
import datetime
import math
import pytesseract
from PIL import Image
def detect_cn(filename):
cfg.TEST.HAS_RPN = True # Use RPN for proposals
args = parse_args()
prototxt = os.path.join(cfg.MODELS_DIR, NETS[args.demo_net][0],
'faster_rcnn_alt_opt', 'faster_rcnn_test.pt')
caffemodel = os.path.join(cfg.DATA_DIR, 'faster_rcnn_models',
NETS[args.demo_net][1])
if not os.path.isfile(caffemodel):
raise IOError(('{:s} not found.\nDid you run ./data/script/'
'fetch_faster_rcnn_models.sh?').format(caffemodel))
if args.cpu_mode:
caffe.set_mode_cpu()
else:
caffe.set_mode_gpu()
caffe.set_device(args.gpu_id)
cfg.GPU_ID = args.gpu_id
net = caffe.Net(prototxt, caffemodel, caffe.TEST)
print '\n\nLoaded network {:s}'.format(caffemodel)
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'Demo for CN image...'
return demo2(net, filename)
Do you think i should load all those librairies in modules of job server.submit?
I want to use the pp bacause detect_cn takes 2 minutes to give results
any ideas?
Yes, you should import all these modules when wou submit your function into the execution queue.
f1 = job_server.submit(detect_cn, (filename,),modules=("math","numpy", ...))