We Keep Coding

Python scripting with ete3 to query NCBI's Taxonomy: "sqlite3 Warning (can only execute one statement at a time)"

I am using this script:
import csv
import time
import sys
from ete3 import NCBITaxa
ncbi = NCBITaxa()
def get_desired_ranks(taxid, desired_ranks):
lineage = ncbi.get_lineage(taxid)
names = ncbi.get_taxid_translator(lineage)
lineage2ranks = ncbi.get_rank(names)
ranks2lineage = dict((rank,taxid) for (taxid, rank) in lineage2ranks.items())
return{'{}_id'.format(rank): ranks2lineage.get(rank, '<not present>') for rank in desired_ranks}
if __name__ == '__main__':
file = open(sys.argv[1], "r")
taxids = []
contigs = []
for line in file:
line = line.split("\n")[0]
taxids.append(line.split(",")[0])
contigs.append(line.split(",")[1])
desired_ranks = ['superkingdom', 'phylum']
results = list()
for taxid in taxids:
results.append(list())
results[-1].append(str(taxid))
ranks = get_desired_ranks(taxid, desired_ranks)
for key, rank in ranks.items():
if rank != '<not present>':
results[-1].append(list(ncbi.get_taxid_translator([rank]).values())[0])
else:
results[-1].append(rank)
i = 0
for result in results:
print(contigs[i] + ','),
print(','.join(result))
i += 1
file.close()
The script takes taxids from a file and fetches their respective lineages from a local copy of NCBI's Taxonomy database. Strangely, this script works fine when I run it on small sets of taxids (~70, ~100), but most of my datasets are upwards of 280k taxids and these break the script.
I get this complete error:
Traceback (most recent call last):
File "/data1/lstout/blast/scripts/getLineageByETE3.py", line 31, in <module>
ranks = get_desired_ranks(taxid, desired_ranks)
File "/data1/lstout/blast/scripts/getLineageByETE3.py", line 11, in get_desired_ranks
lineage = ncbi.get_lineage(taxid)
File "/data1/lstout/.local/lib/python2.7/site-packages/ete3/ncbi_taxonomy/ncbiquery.py", line 227, in get_lineage
result = self.db.execute('SELECT track FROM species WHERE taxid=%s' %taxid)
sqlite3.Warning: You can only execute one statement at a time.
The first two files from the traceback are simply the script I referenced above, the third file is one of ete3's. And as I stated, the script works fine with small datasets.
What I have tried:
Importing the time module and sleeping for a few milliseconds/hundredths of a second before/after my offending lines of code on lines 11 and 31. No effect.
Went to line 227 in ete3's code...
result = self.db.execute('SELECT track FROM species WHERE taxid=%s' %merged_conversion[taxid])
and changed the "execute" function to "executescript" in order to be able to handle multiple queries at once (as that seems to be the problem). This produced a new error and led to a rabbit hole of me changing minor things in their script trying to fudge this to work. No result. This is the complete offending function:
def get_lineage(self, taxid):
"""Given a valid taxid number, return its corresponding lineage track as a
hierarchically sorted list of parent taxids.
"""
if not taxid:
return None
result = self.db.execute('SELECT track FROM species WHERE taxid=%s' %taxid)
raw_track = result.fetchone()
if not raw_track:
#perhaps is an obsolete taxid
_, merged_conversion = self._translate_merged([taxid])
if taxid in merged_conversion:
result = self.db.execute('SELECT track FROM species WHERE taxid=%s' %merged_conversion[taxid])
raw_track = result.fetchone()
# if not raise error
if not raw_track:
#raw_track = ["1"]
raise ValueError("%s taxid not found" %taxid)
else:
warnings.warn("taxid %s was translated into %s" %(taxid, merged_conversion[taxid]))
track = list(map(int, raw_track[0].split(",")))
return list(reversed(track))
What bothers me so much is that this works on small amounts of data! I'm running these scripts from my school's high performance computer and have tried running on their head node and in an interactive moab scheduler. Nothing has helped.

Python TypeError: '_sre.SRE_Match' object has no attribute '__getitem__'

Im brand new to python and coding, im trying to get below working.
this is test code and if I can get this working I should be able to build on it.
Like I said im new to this so sorry if its a silly mistake.
# coding=utf-8
import ops # Import the OPS module.
import sys # Import the sys module.
import re
# Subscription processing function
def ops_condition (o):
enter code herestatus, err_str = o.timer.relative("tag",10)
return status
def ops_execute (o):
handle, err_desp = o.cli.open()
print("OPS opens the process of command:",err_desp)
result, n11, n21 = o.cli.execute(handle,"return")
result, n11, n21 = o.cli.execute(handle,"display interface brief | include Ethernet0/0/1")
match = re.search(r"Ethernet0/0/1\s*(\S+)\s*", result)
if not match:
print("Could not determine the state.")
return 0
physical_state = match[1] # Gets the first group from the match.
print (physical_state)
if physical_state == "down":
print("down")
result = o.cli.close(handle)
else :
print("up")
return 0
Error
<setup>('OPS opens the process of command:', 'success')
Oct 17 2018 11:53:39+00:00 setup %%01OPSA/3/OPS_RESULT_EXCEPTION(l)[4]:Script is test.py, current event is tag, instance is 1515334652, exception reason is Trac eback (most recent call last):
File ".lib/frame.py", line 114, in <module>
ret = m.ops_execute(o)
File "flash:$_user/test.py", line 22, in ops_execute
physical_state = match[1] # Gets the first group from the match.
TypeError: '_sre.SRE_Match' object has no attribute '__getitem__'

The __getitem__ method for the regex match objects was only added since Python 3.6. If you're using an earlier version, you can use the group method instead.
Change:
physical_state = match[1]
to:
physical_state = match.group(1)
Please refer to the documentation for details.

Traceback (most recent call last): File "newdata.py", line 51, in <module> temp_res.update(i['statistics']) KeyError: 'statistics'

search_response = youtube.search().list(q=options.q, type='video',
part='id,snippet', maxResults=options.max_results).execute()
videos = {}
# Add each result to the appropriate list, and then display the lists of
# matching videos.
# Filter out channels, and playlists.
for search_result in search_response.get('items', []):
if search_result['id']['kind'] == 'youtube#video':
# videos.append("%s" % (search_result["id"]["videoId"]))
videos[search_result['id']['videoId']] = search_result['snippet'
]['title']
# print "Videos:\n", "\n".join(videos), "\n"
s = ','.join(videos.keys())
videos_list_response = youtube.videos().list(id=s,
part='id,statistics,snippet').execute()
res = []
for i in videos_list_response['items']:
tempres = dict(v_id=i['id'], v_title=videos[i['id']])
tempres.update(i['snippet'])
tempres.update(i['statistics'])
res.append(tempres)
Please enter the search term for videos: football
Your search term is: football
Traceback (most recent call last):
File "data_pull.py", line 61, in
tempres.update(i['statistics'])
KeyError: 'statistics'
Need help to solve this error. got this code from https://www.analyticsvidhya.com/blog/2014/09/mining-youtube-python-social-media-analysis/#comment-126365 and not able to solve this error.
this file ran before error free, but after 4-5 days this error has popped up without changing the code.
Its weird but help would be appreciated!
Thanks

Because with some videos, you cannot get its statistics.
Try with Search term = Lac Troi Son Tung MTP and you will find out this video does not have statistics attribute.

Weird TypeError in jython/python?

I'm bukkit jython/python plugin programmer. Last few days, I'm struggling with this problem. I have to add an potion effect to an user, it's not problem if I enter effect name, duration and amplifier manually in code, but when I want to get them from config, I get this error:
13:38:20 [SEVERE] Could not pass event PlayerInteractEvent to ItemEffect v1.0
Traceback (most recent call last):
File "<iostream>", line 126, in onPlayerInteractEvent
TypeError: addPotionEffect(): 1st arg can't be coerced to org.bukkit.potion.Poti
onEffect
Here's that part of code:
effectname = section.getString("%s.effect"%currentKey)
duration = section.getInt("%s.duration"%currentKey)
durationinticks = duration * 20
geteffectname = "PotionEffectType.%s"%effectname
getpotioneffect = "PotionEffect(%s, %i, 1)"%(geteffectname, durationinticks)
geteffectname = "PotionEffectType.%s"%effectname
if iteminhand == currentKey:
event.getPlayer().addPotionEffect(getpotioneffect)
When I print getpotioneffect out, I get:
13:38:20 [INFO] PotionEffect(PotionEffectType.SPEED, 600, 1)
which is okay, and should work. I tested it without getting informations from config, and it works perfectly... To sum up, code above is not working, but below one works:
getpotioneffect = PotionEffect(PotionEffectType.SPEED, 600, 1)
if iteminhand == currentKey:
event.getPlayer().addPotionEffect(getpotioneffect)
Link to javadocs of this event!
http://jd.bukkit.org/rb/apidocs/org/bukkit/entity/LivingEntity.html#addPotionEffect(org.bukkit.potion.PotionEffect)
Thanks!

In your first snippet, getpotioneffect is a string. You can check it adding print type(getpotioneffect) somewhere.
What you want is to replace this :
geteffectname = "PotionEffectType.%s"%effectname
getpotioneffect = "PotionEffect(%s, %i, 1)"%(geteffectname, durationinticks)
with this:
effect_type = getattr(PotionEffectType, effectname)
potion_effect = PotionEffect(effect_type, durationinticks, 1)

Python + GStreamer - Won't connect

I'm having trouble combining audio and video into one file. The Python code looks like this;
filmPipe = gst.Pipeline("filmPipe")
filmSrc = gst.element_factory_make("multifilesrc", "filmSrc")
filmSrc.set_property("location", "pictures/%d.png")
filmFilt1 = gst.element_factory_make("capsfilter", "filmFilt1")
filmCap1 = gst.Caps("image/png,framerate=5/1,pixel-aspect-ratio=1/1")
filmFilt1.set_property("caps", filmCap1)
filmPngDec = gst.element_factory_make("pngdec", "filmPngDec")
filmff = gst.element_factory_make("ffmpegcolorspace", "filmff")
filmFilt2 = gst.element_factory_make("capsfilter", "filmFilt2")
filmCap2 = gst.Caps("video/x-raw-yuv")
filmFilt2.set_property("caps", filmCap2)
filmTheora = gst.element_factory_make("xvidenc", "filmTheora")
filmQue = gst.element_factory_make("queue", "filmQue")
filmOggmux = gst.element_factory_make("ffmux_mp4", "filmOggmux")
filmFilesink = gst.element_factory_make("filesink", "filmFilesink")
filmFilesink.set_property("location", self.movPath)
musicSrc = gst.element_factory_make("filesrc", "musicSrc")
musicSrc.set_property("location", self.musicPath)
musicDec = gst.element_factory_make("ffdec_mp3", "musicDec")
musicEnc = gst.element_factory_make("lame", "musicEnc")
musicQue = gst.element_factory_make("queue", "musicQue")
filmPipe.add(filmSrc, filmFilt1, filmPngDec, filmff, filmFilt2, filmTheora, filmQue, filmOggmux, filmFilesink)
filmPipe.add(musicSrc, musicDec, musicEnc, musicQue)
gst.element_link_many(filmSrc, filmFilt1, filmPngDec, filmff, filmFilt2, filmTheora, filmQue, filmOggmux, filmFilesink)
gst.element_link_many(musicSrc, musicDec, musicEnc, musicQue, filmOggmux, filmFilesink)
filmPipe.set_state(gst.STATE_PLAYING)
This returns the following error:
Traceback (most recent call last):
File "app.py", line 100, in movGen
gst.element_link_many(musicSrc, musicDec, musicEnc, musicQue, filmOggmux, filmFilesink)
gst.LinkError: failed to link filmOggmux with filmFilesink
Does anybody know where I'm going wrong, or how to fix this?

You are linking 2 times filmOggmux to filmFilesink: this is not allowed, only one link is possible.
Try removing filmFilesink in the second gst.element_link_many().