I am trying to batch run workspaces from a standalone python script and it fails. I am particularly wondering if I am passing the parameters correctly. I was using an exception catch with a message but it actually provided less information:
'code' Traceback (most recent call last):
File "C:\Users\YouDoWellItDoesGood\Downloads \SixtyFourTest.py", line 89, in <module>
wrkRunner.runWithParameters(qcWorkspace,parameters)
FMEException: FMEException: 1: Failure running workspace 'C:\Users\YouDoWellItDoesGood\Downloads\Anadarko_QC_Tool.fmwt' 'code'
The code:
# Note:
# The path to fmeobjects must by in your python
# path so you may need something like this:
import sys
sys.path.append(r"C:\Program Files\FME\fmeobjects\python27")
import fmeobjects
import fnmatch
import xlrd
import arcpy
import datetime
import os
##dictionary to hold the day of week value generated by the statusRunDateNum
##variable. the first
##number in the dictionary entery is the day of the week as a number. The other
##two numbers are the column numbers where we will find our project status values
datColumnDict={1:[2,3],2:[5,6],3:[8,9],4:[11,12],5:[14,15]}
qcWorkspace=r"C:\Users\YouDoWellItDoesGood\Downloads\Anadarko_QC_Tool.fmwt"
#will return a number depending what day it is.
statusRunDateNum=datetime.datetime.today().isoweekday()
# location of the status report/ will be turned into a variable.
PSL=r"C:\Users\YouDoWellItDoesGood\Downloads\Project%20Status%20Worksheet%203_20%20to%203_24.xlsx"
pjStatus=xlrd.open_workbook(PSL)
pjSheet=pjStatus.sheet_by_index(0)
pjList=[]
for row_index in range(1,pjSheet.nrows):
statusColumn=str(pjSheet.cell(row_index,datColumnDict[statusRunDateNum][1]))
statusColumnNoColon=statusColumn.split(":")[1]
projColumn=str(pjSheet.cell(row_index,datColumnDict[statusRunDateNum][0]))
projColumnNoDec=projColumn.split(".")[0]
projColumnNoColon=projColumnNoDec.split(":")[1]
if "QC" in str(statusColumn):
pjList.append(projColumnNoColon)
print statusColumnNoColon
print pjList
pjDir=r"F:\Projects\ANADARKO-DELAWARE BASIN"
pjDirList=os.listdir(pjDir)
wrkspc=[]
for x in range(len(pjList)):
#create string from TGS proj number
match=str(pjList[x])+"*"
#declare feature list
for f in pjDirList:
if fnmatch.fnmatch(f,match):
## print match[:-1]
gisPath=os.path.join(pjDir,f,"GIS")
for gf in os.listdir(gisPath):
if gf.startswith("DB_") or gf.startswith("ID") and gf.endswith(".gdb"):
wrkspc.append(os.path.join(gisPath,gf))
print wrkspc
for w in wrkspc:
parameters={}
parameters['_gdbpath']=w
parameters['_user']="test"
parameters['_with_corrections']='False'
#parameters['FEATURE_TYPES']="WE_PT WE_PATH_LN WE_PAD_POLY WD_PT WD_POLY WD_LN VG_PT VG_POLY VG_LN VG_BUFFER_POLY TR_TRANS_OTHER_PT TR_TRANS_OTHER_POLY TR_TRANS_OTHER_LN TR_RUNWAY_POLY TR_ROAD_LOW_WATER_PT TR_ROAD_CENTER_LN TR_RAILROAD_PT TR_RAILROAD_POLY TR_RAILROAD_LN TR_LANDING_ZONE_PT TR_LANDING_ZONE_POLY TR_EDGE_OF_PAVEMENT_LN ST_OTHER_PT ST_OTHER_POLY ST_OTHER_LN SI_FACILITY_PERIMETER_POLY SI_FACILITY_OTHER_POLY RC_WETLANDS_POLY RC_SOILBED_PREPARATION_POLY RC_SOIL_SAMPLE_PT RC_SOIL_AMENDMENT_POLY RC_SEEDBED_PREPARATION_POLY RC_RECLAMATION_OTHER_PT RC_RECLAMATION_OTHER_POLY RC_RECLAMATION_OTHER_LN RC_MULCH_POLY RC_HYDROMULCH_POLY RC_BORROW_PIT_POLY PL_VENT_PIPE_PT PL_TRENCH_BREAKER_LN PL_TEST_LEAD_PT PL_TEE_PT PL_TAP_PT PL_SLEEVE_LN PL_ROUTING_NOTE_PT PL_ROCK_SHIELD_LN PL_REDUCER_PT PL_PUMP_STATION_PT PL_PIPELINE_LN PL_PIPE_BEND_LN PL_PIG_SIGNAL_PT PL_PI_EXCAVATION_PT PL_NAT_GROUND_PT PL_METER_STATION_PT PL_JOIN_PT PL_INJECTOR_PT PL_GIRTH_WELD_PT PL_FLANGE_PT PL_ELBOW_PT PL_DRIP_PT PL_DEPTH_OF_COVER_PT PL_COMPRESSOR_STATION_PT"
wrkRunner=fmeobjects.FMEWorkspaceRunner()
wrkRunner.runWithParameters(qcWorkspace,parameters)
print w
print parameters
Related
I am running the following script:
# STEP 1: import packages, declare tindexes
import pandas as pd
import yfinance as yf
import datetime as dt
bnpl_tindex = ["APT.AX","Z1P.AX","LFS.AX","SZL.AX","HUM.AX","SPT.AX","OPY.AX","IOU.AX","LBY.AX","DOU.AX"]
target_dates = pd.date_range(start=dt.date.today() - dt.timedelta(days=365), end=dt.date.today(), freq="M").strftime('%Y-%m-%d')
target_dates = target_dates.append(pd.Index([dt.date.today().strftime('%Y-%m-%d')]))
target_dates.sort_values()
#STEP 2: source functions
from collect_index_data import collect_index_data
#DELETE LATER... TESTING!
collect_index_data(bnpl_tindex, target_dates)
#
collect_index_data is as follows:
def collect_index_data(ticker_list,date_list):
if (bool(ticker_list) and all(isinstance(elem, str) for elem in ticker_list) )==False:
sys.exit('Input should be a list or a single string.')
else:
print("Components of Index: ")
#initialise dictionary
d = {}
#loop through ticker list
for x in ticker_list:
d["DF.{0}".format(x)] = yf.Ticker(x)
#testing
print(x)
#testing
print(d)
and I get the following error message
Components of Index:
Traceback (most recent call last):
File "C:\Users\thoma\Desktop\Files\Programming\Python\run_tindex_data.py", line 27, in <module>
collect_index_data(bnpl_tindex, target_dates)
File "C:\Users\thoma\Desktop\Files\Programming\Python\collect_index_data.py", line 12, in collect_index_data
d["DF.{0}".format(x)] = yf.Ticker("MSFT")
NameError: name 'yf' is not defined
My question is why is yfinance package not being recognised in my function?
I could import it inside the function, but I plan to run the function multiple times in a script - so this would be computationally wasteful.
thanks!
I am trying to learn how to use the backtrader module on Python. I copied the code directly from the website but receiving an error message.
Here is the website: https://www.backtrader.com/docu/quickstart/quickstart/
I downloaded S&P500 stock data from Yahoo Finance and saved it into an excel file named 'SPY'. Here is the code so far:
from __future__ import (absolute_import, division, print_function,
unicode_literals)
import datetime # For datetime objects
import os.path # To manage paths
import sys # To find out the script name (in argv[0])
# Import the backtrader platform
import backtrader as bt
if __name__ == '__main__':
# Create a cerebro entity
cerebro = bt.Cerebro()
# Datas are in a subfolder of the samples. Need to find where the script is
# because it could have been called from anywhere
modpath = os.path.dirname(os.path.abspath(sys.argv[0]))
datapath = os.path.join(modpath, 'C:\\Users\\xboss\\Desktop\\SPY.csv')
# Create a Data Feed
data = bt.feeds.YahooFinanceCSVData(
dataname=datapath,
# Do not pass values before this date
fromdate=datetime.datetime(2000, 1, 1),
# Do not pass values after this date
todate=datetime.datetime(2000, 12, 31),
reverse=False)
# Add the Data Feed to Cerebro
cerebro.adddata(data)
# Set our desired cash start
cerebro.broker.setcash(100000.0)
# Print out the starting conditions
print('Starting Portfolio Value: %.2f' % cerebro.broker.getvalue())
# Run over everything
cerebro.run()
# Print out the final result
print('Final Portfolio Value: %.2f' % cerebro.broker.getvalue())
Here is the error that I am receiving:
C:\Users\xboss\PycharmProjects\BackTraderDemo\venv\Scripts\python.exe C:/Users/xboss/PycharmProjects/BackTraderDemo/backtrader_quickstart.py
Traceback (most recent call last):
File "C:/Users/xboss/PycharmProjects/BackTraderDemo/backtrader_quickstart.py", line 39, in <module>
cerebro.run()
File "C:\Users\xboss\PycharmProjects\BackTraderDemo\venv\lib\site-packages\backtrader\cerebro.py", line 1127, in run
runstrat = self.runstrategies(iterstrat)
File "C:\Users\xboss\PycharmProjects\BackTraderDemo\venv\lib\site-packages\backtrader\cerebro.py", line 1212, in runstrategies
data.preload()
File "C:\Users\xboss\PycharmProjects\BackTraderDemo\venv\lib\site-packages\backtrader\feed.py", line 688, in preload
while self.load():
Starting Portfolio Value: 100000.00
File "C:\Users\xboss\PycharmProjects\BackTraderDemo\venv\lib\site-packages\backtrader\feed.py", line 479, in load
_loadret = self._load()
File "C:\Users\xboss\PycharmProjects\BackTraderDemo\venv\lib\site-packages\backtrader\feed.py", line 710, in _load
return self._loadline(linetokens)
File "C:\Users\xboss\PycharmProjects\BackTraderDemo\venv\lib\site-packages\backtrader\feeds\yahoo.py", line 129, in _loadline
dt = date(int(dttxt[0:4]), int(dttxt[5:7]), int(dttxt[8:10]))
ValueError: invalid literal for int() with base 10: '1/29'
Process finished with exit code 1
Does anyone have any suggestions? Any help would be greatly appreciated. Thank you so much for your time!
You get the error because of using custom csv with YahooFinanceCSVData method.
You should import them using GenericCSVData method.
data = btfeed.GenericCSVData(
dataname='SPY.csv',
fromdate=datetime.datetime(2000, 1, 1),
todate=datetime.datetime(2000, 12, 31),
nullvalue=0.0,
dtformat=('%Y-%m-%d'),
datetime=0,
high=1,
low=2,
open=3,
close=4,
volume=5,
openinterest=-1
)
For more information you can see the instruction here
I am using this script:
import csv
import time
import sys
from ete3 import NCBITaxa
ncbi = NCBITaxa()
def get_desired_ranks(taxid, desired_ranks):
lineage = ncbi.get_lineage(taxid)
names = ncbi.get_taxid_translator(lineage)
lineage2ranks = ncbi.get_rank(names)
ranks2lineage = dict((rank,taxid) for (taxid, rank) in lineage2ranks.items())
return{'{}_id'.format(rank): ranks2lineage.get(rank, '<not present>') for rank in desired_ranks}
if __name__ == '__main__':
file = open(sys.argv[1], "r")
taxids = []
contigs = []
for line in file:
line = line.split("\n")[0]
taxids.append(line.split(",")[0])
contigs.append(line.split(",")[1])
desired_ranks = ['superkingdom', 'phylum']
results = list()
for taxid in taxids:
results.append(list())
results[-1].append(str(taxid))
ranks = get_desired_ranks(taxid, desired_ranks)
for key, rank in ranks.items():
if rank != '<not present>':
results[-1].append(list(ncbi.get_taxid_translator([rank]).values())[0])
else:
results[-1].append(rank)
i = 0
for result in results:
print(contigs[i] + ','),
print(','.join(result))
i += 1
file.close()
The script takes taxids from a file and fetches their respective lineages from a local copy of NCBI's Taxonomy database. Strangely, this script works fine when I run it on small sets of taxids (~70, ~100), but most of my datasets are upwards of 280k taxids and these break the script.
I get this complete error:
Traceback (most recent call last):
File "/data1/lstout/blast/scripts/getLineageByETE3.py", line 31, in <module>
ranks = get_desired_ranks(taxid, desired_ranks)
File "/data1/lstout/blast/scripts/getLineageByETE3.py", line 11, in get_desired_ranks
lineage = ncbi.get_lineage(taxid)
File "/data1/lstout/.local/lib/python2.7/site-packages/ete3/ncbi_taxonomy/ncbiquery.py", line 227, in get_lineage
result = self.db.execute('SELECT track FROM species WHERE taxid=%s' %taxid)
sqlite3.Warning: You can only execute one statement at a time.
The first two files from the traceback are simply the script I referenced above, the third file is one of ete3's. And as I stated, the script works fine with small datasets.
What I have tried:
Importing the time module and sleeping for a few milliseconds/hundredths of a second before/after my offending lines of code on lines 11 and 31. No effect.
Went to line 227 in ete3's code...
result = self.db.execute('SELECT track FROM species WHERE taxid=%s' %merged_conversion[taxid])
and changed the "execute" function to "executescript" in order to be able to handle multiple queries at once (as that seems to be the problem). This produced a new error and led to a rabbit hole of me changing minor things in their script trying to fudge this to work. No result. This is the complete offending function:
def get_lineage(self, taxid):
"""Given a valid taxid number, return its corresponding lineage track as a
hierarchically sorted list of parent taxids.
"""
if not taxid:
return None
result = self.db.execute('SELECT track FROM species WHERE taxid=%s' %taxid)
raw_track = result.fetchone()
if not raw_track:
#perhaps is an obsolete taxid
_, merged_conversion = self._translate_merged([taxid])
if taxid in merged_conversion:
result = self.db.execute('SELECT track FROM species WHERE taxid=%s' %merged_conversion[taxid])
raw_track = result.fetchone()
# if not raise error
if not raw_track:
#raw_track = ["1"]
raise ValueError("%s taxid not found" %taxid)
else:
warnings.warn("taxid %s was translated into %s" %(taxid, merged_conversion[taxid]))
track = list(map(int, raw_track[0].split(",")))
return list(reversed(track))
What bothers me so much is that this works on small amounts of data! I'm running these scripts from my school's high performance computer and have tried running on their head node and in an interactive moab scheduler. Nothing has helped.
Im brand new to python and coding, im trying to get below working.
this is test code and if I can get this working I should be able to build on it.
Like I said im new to this so sorry if its a silly mistake.
# coding=utf-8
import ops # Import the OPS module.
import sys # Import the sys module.
import re
# Subscription processing function
def ops_condition (o):
enter code herestatus, err_str = o.timer.relative("tag",10)
return status
def ops_execute (o):
handle, err_desp = o.cli.open()
print("OPS opens the process of command:",err_desp)
result, n11, n21 = o.cli.execute(handle,"return")
result, n11, n21 = o.cli.execute(handle,"display interface brief | include Ethernet0/0/1")
match = re.search(r"Ethernet0/0/1\s*(\S+)\s*", result)
if not match:
print("Could not determine the state.")
return 0
physical_state = match[1] # Gets the first group from the match.
print (physical_state)
if physical_state == "down":
print("down")
result = o.cli.close(handle)
else :
print("up")
return 0
Error
<setup>('OPS opens the process of command:', 'success')
Oct 17 2018 11:53:39+00:00 setup %%01OPSA/3/OPS_RESULT_EXCEPTION(l)[4]:Script is test.py, current event is tag, instance is 1515334652, exception reason is Trac eback (most recent call last):
File ".lib/frame.py", line 114, in <module>
ret = m.ops_execute(o)
File "flash:$_user/test.py", line 22, in ops_execute
physical_state = match[1] # Gets the first group from the match.
TypeError: '_sre.SRE_Match' object has no attribute '__getitem__'
The __getitem__ method for the regex match objects was only added since Python 3.6. If you're using an earlier version, you can use the group method instead.
Change:
physical_state = match[1]
to:
physical_state = match.group(1)
Please refer to the documentation for details.
I try to interpolate irregular data to monthly resolution. the error is like this
enter image description here
File "/usr/local/lib/python2.7/dist-packages/traces/timeseries.py", line 122, in _get_linear_interpolate
return self.default
File "/usr/local/lib/python2.7/dist-packages/traces/timeseries.py", line 106, in default
raise KeyError(msg)
KeyError: "can't get value without a measurement (or a default)"
the code is like this:
import datetime
from datetime import timedelta
import traces
import re
import os
ls = []
def cal_time_series(_ff):
with open(_ff) as _fi:
for _l in _fi.read().splitlines():
_vv = _l.split(',')
if _vv[0] == 'date':
continue
_date = _vv[0].split('-')
ls.append((datetime.datetime(int(_date[0]),int(_date[1]),int(_date[2])),int(_vv[1])))
ts = traces.TimeSeries(ls)
ts.sample(sampling_period=datetime.timedelta(days=30),start=datetime.datetime(2000,1,15),end=datetime.datetime(2015,12,15),interpolate='linear',)
cal_time_series('/mnt/d/thesis/code/point1.csv')
There is an indentation problem with your code:
if _vv[0] == 'date':
continue
_date = _vv[0].split('-')
ls.append((datetime.datetime(int(_date[0]),int(_date[1]),int(_date[2])),int(_vv[1])))
The bottom two lines will never get hit, because the iteration continues immediately prior.