As part of the testing regime for building a python application [a Jupyter notebook, in this instance], I want to test that all the modules installed in the system will actually load.
import module from string variable pointed me to build the following code:
import re
import subprocess
s_counter = 0
f_counter = 0
errors = []
libraries = []
output = subprocess.run(['conda', 'list'], stdout=subprocess.PIPE).stdout.decode('utf-8').splitlines()
for line in output:
if not re.match('\s*#', line):
words = re.split('\s+', line)
if words[0] != 'python':
libraries.append(words[0])
for lib in libraries:
try:
lib_obj = __import__(lib)
globals()[lib] = lib_obj
s_counter += 1
except ImportError:
errors.append("ERROR: missing python library: " + lib)
f_counter += 1
print(f"Successfuly loaded {s_counter} libraries, failed to load {f_counter}")
print(f"{errors}")
The problem with this simplistic solution is that it assumes the module name is the same as the import name.... and often it isn't.
For example: .... install beautifulsoup4 and from bs4 import BeautifulSoup
So the question is, how do I find out the import string for a module... programatically?
Related
I want to use Jasper Reports with a python-flask app.
I have followed the installation instructions on this page:
https://pypi.org/project/pyreportjasper/
It fails on the line import jpy, I am getting the error:
ImportError: DLL load failed while importing jpy: The specified module could not be found.
I am using 32 bit python Python: 3.8.6rc1.
32 bit JDK: javac 1.8.0_261
When I run pip install pyreportjasper, it says: Requirement already satisfied: jpy in c:\users....
Any ideas?
It seems that you need to import jpyutil and then initialize the JVM prior to calling import jpy. The following fixed this error for me. One step closer.
import jpyutil
jpyutil.init_jvm(jvm_maxmem='512M')
import jpy
UPDATE: so while the above statement is true, it doesn't directly address the issue of running the example code at the project page: https://pypi.org/project/pyreportjasper/
As I dug into the code I found that jasperpy.py is attempting to import jpy BEFORE the __init__ method is run and therefore BEFORE the JVM is initialized via the jpyutil.init_jvm() method call. So I edited jasperpy.py and moved import jpy from line 14 to line 56, directly after the call to jpyutil.init_jvm() and before the call to jpy.get_type('java.io.File') and I am able to successfully run Jasper reports from python via the example code shown. Here is what my jasperpy.py looks like now.
# -*- coding: utf-8 -*-
# GNU GENERAL PUBLIC LICENSE
#
# Copyright (c) 2020 Jadson Bonfim Ribeiro <contato#jadsonbr.com.br>
#
import os
import subprocess
import re
import xml.etree.ElementTree as ET
import tempfile
import jpyutil
#import jpy
import json
from requests import Request, Session
FORMATS = (
'pdf',
'rtf',
'xls',
'xlsx',
'docx',
'odt',
'ods',
'pptx',
'csv',
'html',
'xhtml',
'xml',
'jrprint',
)
EXECUTABLE = 'jasperstarter'
class JasperPy:
_FORMATS_JSON = ('pdf')
_FORMATS_METHODS_REQUEST = ('GET', 'POST', 'PUT')
def __init__(self, resource_dir=False, jvm_maxmem='512M', jvm_classpath=None):
self.WINDOWS = True if os.name == 'nt' else False
self.SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
self.LIBS = os.path.join(self.SCRIPT_DIR, 'jasperstarter', 'lib')
if not os.path.isdir(self.LIBS):
raise NameError('Unable to find lib in {0}'.format(self.LIBS))
self.CLASSPATH = os.path.join(self.LIBS, 'jasperstarter.jar')
if not os.path.exists(self.CLASSPATH):
raise NameError('Unable to find jasperstarter in {0}'.format(self.LIBS))
if jvm_classpath is None:
jpyutil.init_jvm(jvm_maxmem=jvm_maxmem, jvm_classpath=[self.CLASSPATH])
else:
jpyutil.init_jvm(jvm_maxmem=jvm_maxmem, jvm_classpath=[self.CLASSPATH, jvm_classpath])
# IMPORT jpy HERE AFTER init_jvm
import jpy
self.jvFile = jpy.get_type('java.io.File')
self.jvArrays = jpy.get_type('java.util.Arrays')
self.jvReport = jpy.get_type('de.cenote.jasperstarter.Report')
self.jvConfig = jpy.get_type('de.cenote.jasperstarter.Config')
self.jvDsType = jpy.get_type('de.cenote.jasperstarter.types.DsType')
self.jvApplicationClasspath = jpy.get_type('de.cenote.tools.classpath.ApplicationClasspath')
self.jvHashMap = jpy.get_type('java.util.HashMap')
self.jvLocale = jpy.get_type('java.util.Locale')
self.jvJasperFillManager = jpy.get_type('net.sf.jasperreports.engine.JasperFillManager')
self.jvDb = jpy.get_type('de.cenote.jasperstarter.Db')
self.jvJsonQueryExecuterFactory = jpy.get_type('net.sf.jasperreports.engine.query.JsonQueryExecuterFactory')
self.jvJasperExportManager = jpy.get_type('net.sf.jasperreports.engine.JasperExportManager')
.
.
.
.
This file is located in your Python directory under Lib\site-packages\pyreportjasper-2.0.2-py3.8.egg\pyreportjasper. While this is currently a hack to make it work, there obviously needs to be a fix to the package which I will attempt to address on the pyreportjasper github page.
I'm tyring to import the "gleam" package in Python 3. I have installed the "gleam" package successfully, but still it showing error.
from wtforms import fields
from ggplot import *
from gleam import Page, panels
class ScatterInput(panels.Inputs):
title = fields.StringField(label="Title of plot:")
yvar = fields.SelectField(label="Y axis",
choices=[("beef", "Beef"),
("pork", "Pork")])
smoother = fields.BooleanField(label="Smoothing Curve")
class ScatterPlot(panels.Plot):
name = "Scatter"
def plot(self, inputs):
p = ggplot(meat, aes(x='date', y=inputs.yvar))
if inputs.smoother:
p = p + stat_smooth(color="blue")
p = p + geom_point() + ggtitle(inputs.title)
return p
class ScatterPage(Page):
input = ScatterInput()
output = ScatterPlot()
ScatterPage.run()
Error:
ModuleNotFoundError - Traceback (most> recent call last) in ()
----> 1 import gleam
C:\pythonNJ\lib\site-packages\gleam__init__.py in ()
5 import os
6 import json
----> 7 import urlparse
8 from collections import namedtuple
9
ModuleNotFoundError: No module named 'urlparse'
I looked for the solution and I found that urlparse has been moved to a new module in python 3, which can be imported as
from urllib.parse import urlparse
And I even imported it, but still when I trying to import "gleam" package it shows error of module "urlparse". Can you suggest me how to bypass it (bypassing import urlparse statement and importing gleam package in Python 3).
I know how to import the urlparse but I don't know how to import the gleam package.
You have two possiblities:
Modify source code yourself as you stated inside gleam package, but it could work incorrectly.
Fall back to version of python it works on - so 2.7 it seems, since the modification you mentioned was done with python 3.0 release. It's stated in docs here.
Just do this to get over it:
from:
import urlparser
to:
import urllib.parse
I have a module name as a string (e.g. 'logging') that was given by querying the module attribute of an object.
How can I differentiate between modules that are part of my project and modules that are part of python standard library?
I know that I can check if this module was installed by pip using pip.get_installed_distributions(), but these are not related to the standard library
Note: I'm working on python 2.7 so solutions that are valid only in python 3.x are less relevant.
Unlike the answer here, I was looking for a solution that can be run in O(1) and will not require holding an array of results nor having to scan the directory for every query.
Thanks.
Quick 'n dirty solution, using the standard module imp:
import imp
import os.path
import sys
python_path = os.path.dirname(sys.executable)
my_mod_name = 'logging'
module_path = imp.find_module(my_mod_name)[1]
if 'site-packages' in module_path or python_path in module_path or not imp.is_builtin(my_mod_name):
print('module', my_mod_name, 'is not included in standard python library')
EDIT:
I used the solution which is here.
import distutils.sysconfig as sysconfig
import os
def std_modules():
ret_list = []
std_lib = sysconfig.get_python_lib(standard_lib=True)
for top, dirs, files in os.walk(std_lib):
for nm in files:
if nm != '__init__.py' and nm[-3:] == '.py':
ret_list.append(os.path.join(top, nm)[len(std_lib)+1:-3].replace('\\','.'))
return ret_list
l = std_modules()
print("logging" in l)
print("os" in l)
Output:
False
True
This works in both Python 2 and Python 3.
BEFORE EDIT:
I guess, you can use Python Docs. Here are standard library parts of Python 2 Docs and Python 3 Docs. Also, you can select the exact version of Python.
None of the solutions above were what I wanted, so I did it yet another way. Posting here in case it's useful to anyone.
import os
def standard_lib_names_gen(include_underscored=False):
standard_lib_dir = os.path.dirname(os.__file__)
for filename in os.listdir(standard_lib_dir):
if not include_underscored and filename.startswith('_'):
continue
filepath = os.path.join(standard_lib_dir, filename)
name, ext = os.path.splitext(filename)
if filename.endswith('.py') and os.path.isfile(filepath):
if str.isidentifier(name):
yield name
elif os.path.isdir(filepath) and '__init__.py' in os.listdir(filepath):
yield name
>>> standard_lib_names = set(standard_lib_names_gen(include_underscored=True))
>>> # verify that a few known libs are there (including three folders and three py files)
>>> assert {'collections', 'asyncio', 'os', 'dis', '__future__'}.issubset(standard_lib_names)
>>> # verify that other decoys are not in there
>>> assert {'__pycache__', 'LICENSE.txt', 'config-3.8-darwin', '.DS_Store'}.isdisjoint(standard_lib_names)
>>>
>>> len(standard_lib_names)
200
>>>
>>> # not including underscored
>>> standard_lib_names = set(standard_lib_names_gen(include_underscored=False))
>>> len(standard_lib_names)
184
There is also a good answer in an old duplicate of this post: https://stackoverflow.com/a/28873415/7262247
This is how you can do it (you need to pip install stdlib_list first):
from stdlib_list import stdlib_list
import sys
all_stdlib_symbols = stdlib_list('.'.join([str(v) for v in sys.version_info[0:2]]))
module_name = 'collections'
if module_name in all_stdlib_symbols:
print("%s is in stdlib" % module_name)
File setup:
...\Project_Folder
...\Project_Folder\Project.py
...\Project_folder\Script\TestScript.py
I'm attempting to have Project.py import modules from the folder Script based on user input.
Python Version: 3.4.2
Ideally, the script would look something like
q = str(input("Input: "))
from Script import q
However, python does not recognize q as a variable when using import.
I've tried using importlib, however I cannot figure out how to import from the Script folder mentioned above.
import importlib
q = str(input("Input: "))
module = importlib.import_module(q, package=None)
I'm not certain where I would implement the file path.
Repeat of my answer originally posted at How to import a module given the full path?
as this is a Python 3.4 specific question:
This area of Python 3.4 seems to be extremely tortuous to understand, mainly because the documentation doesn't give good examples! This was my attempt using non-deprecated modules. It will import a module given the path to the .py file. I'm using it to load "plugins" at runtime.
def import_module_from_file(full_path_to_module):
"""
Import a module given the full path/filename of the .py file
Python 3.4
"""
module = None
try:
# Get module name and path from full path
module_dir, module_file = os.path.split(full_path_to_module)
module_name, module_ext = os.path.splitext(module_file)
# Get module "spec" from filename
spec = importlib.util.spec_from_file_location(module_name,full_path_to_module)
module = spec.loader.load_module()
except Exception as ec:
# Simple error printing
# Insert "sophisticated" stuff here
print(ec)
finally:
return module
# load module dynamically
path = "<enter your path here>"
module = import_module_from_file(path)
# Now use the module
# e.g. module.myFunction()
I did this by defining the entire import line as a string, formatting the string with q and then using the exec command:
imp = 'from Script import %s' %q
exec imp
I'm in a pickle with writing a script that can SSH into device, run a command and parse that data out to a file. I've written this using Pyparsing and Exscript then I found out that the device I'm going to be using this on is using Python 2.4.4 and Debian 4.1.1 so the modules will not work on this. Now I am back to the drawing board trying to find out how to do this with NO modules. Anyone have any reference or point me in the right direction for this? Thank you in advance.
Here is my code:
from Exscript.util.interact import read_login
from Exscript.protocols import SSH2
import uuid
from pyparsing import *
import re
import yaml
account = read_login()
conn = SSH2()
conn.connect('172.0.0.1')
conn.login(account)
conn.execute('foobar')
data = conn.response
conn.send('exit\r')
conn.close()
###### PARSER ######
date_regex = re.compile(r'\d\d-\d\d-\d\d')
time_regex = re.compile(r'\d\d:\d\d:\d\d')
pairs = [{'category': 'General Information',
'kv': Group(Word(alphanums) + Word(alphanums))},
{'category': 'Last Reset:',
'kv': Group(Word(alphas, max=1) + Word(alphas)) + Literal(':').suppress()
+ Group(Regex(date_regex) + Regex(time_regex)
+ Optional(SkipTo(LineEnd())))
}
]
# build list of categories with associated parsing rules
categories = [Word("# ").suppress() + x['category']
+ OneOrMore(Group(x['kv']))
for x in pairs]
# account for thing you don't have specific rules for
categories.append(Word("#").suppress() + Optional(SkipTo(LineEnd())) +
Group(OneOrMore(Combine(Word(alphanums) + SkipTo(LineEnd()))))
)
# OR all the categories together
categories_ored = categories[0]
for c in categories[1:]:
categories_ored |= c
configDef = OneOrMore(categories_ored)
suppress_tokens = ["show all", "SSH>", "Active System Configuration"]
suppresses = [Literal(x).suppress() for x in suppress_tokens]
for s in suppresses:
configDef.ignore(s)
result = configDef.parseString(data)
for e in result:
print(e)
with open('/Users/MyMac/development/data.yml', 'w') as outfile:
outfile.write( yaml.dump(e))
UPDATE
I have followed the advice below and now have Pexpect installed and found a older version of Python-Pyparsing that I have also installed. So I'm on my way again to getting my scripts to work with modules. Thanks!
Looks like this is already solved, but...
As long as your SSH is configured for this host (or the host doesn't require you to log-in), you should be able to do the following.
import os
""" This will execute foobar on the remote host
and store the command output to a text file
on your machine."""
os.system("ssh 172.0.0.1 foobar > ~/data.txt")
""" Commence processing """
data = open("data.txt", mode='r')
# and so on and so on
You can also use the subprocess library, but os.system for these types of tasks is the simplest IMO.