xml.etree giving an Invalid Token Error When Parsing PDF Metadata - python

I have a project that searches PDFs for URLs and in the process extracts the PDF Metadata. It works perfectly around 99.6% of the time without any errors. But every once in a while, a file throws the old "invalid token error. Traceback Below:
Traceback (most recent call last):
File "c:\python38\lib\runpy.py", line 193, in _run_module_as_main
return run_code(code, main_globals, None,
File "c:\python38\lib\runpy.py", line 86, in run_code
exec(code, run_globals)
File "C:\Python38\Scripts\linkrot.exe_main.py", line 7, in
File "c:\python38\lib\site-packages\linkrot\cli.py", line 182, in main
pdf = linkrot.linkrot(args.pdf)
File "c:\python38\lib\site-packages\linkrot_init.py", line 131, in init
self.reader = PDFMinerBackend(self.stream)
File "c:\python38\lib\site-packages\linkrot\backends.py", line 213, in init
self.metadata.update(xmp_to_dict(metadata))
File "c:\python38\lib\site-packages\linkrot\libs\xmp.py", line 92, in xmp_to_dict
return XmpParser(xmp).meta
File "c:\python38\lib\site-packages\linkrot\libs\xmp.py", line 41, in init
self.tree = ET.XML(xmp)
File "c:\python38\lib\xml\etree\ElementTree.py", line 1320, in XML
parser.feed(text)
xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 55, column 10
My assumption is that there is some sort of issue with the XML extracted from the PDF, but I can't be sure. Is there a workaround? Some way the rest of the program could run when this error throws? The metadata is valuable to the process so I'd like to keep it if possible. I don't know etree that well, so I'd appreciate some help. The Code itself is below:
class XmpParser(object):
"""
Parses an XMP string into a dictionary.
Usage:
parser = XmpParser(xmpstring)
meta = parser.meta
"""
def __init__(self, xmp):
self.tree = ET.XML(xmp)
self.rdftree = self.tree.find(RDF_NS + "RDF")
#property
def meta(self):
""" A dictionary of all the parsed metadata. """
meta = defaultdict(dict)
if self.rdftree:
for desc in self.rdftree.findall(RDF_NS + "Description"):
for (
el
) in (
desc.iter()
):
ns, tag = self._parse_tag(el)
value = self._parse_value(el)
meta[ns][tag] = value
return dict(meta)
def _parse_tag(self, el):
""" Extract the namespace and tag from an element. """
ns = None
tag = el.tag
if tag[0] == "{":
ns, tag = tag[1:].split("}", 1)
if ns in NS_MAP:
ns = NS_MAP[ns]
return ns, tag
def _parse_value(self, el): # noqa: C901
""" Extract the metadata value from an element. """
if el.find(RDF_NS + "Bag") is not None:
value = []
for li in el.findall(RDF_NS + "Bag/" + RDF_NS + "li"):
value.append(li.text)
elif el.find(RDF_NS + "Seq") is not None:
value = []
for li in el.findall(RDF_NS + "Seq/" + RDF_NS + "li"):
value.append(li.text)
elif el.find(RDF_NS + "Alt") is not None:
value = {}
for li in el.findall(RDF_NS + "Alt/" + RDF_NS + "li"):
value[li.get(XML_NS + "lang")] = li.text
else:
value = el.text
return value
Any help or advice would be appreciated.

Related

Prefect is not registering flow when call flow.register() . It triggers can´t pickle error

Using Prefect to register the workflow with my backend server returns this error:
Traceback (most recent call last):
File "/home/usr/Documents/Test_Automation/Automation/qa-automation/date_validator_Prefect/date_validator.py", line 114, in
flow.register(project_name="QA-Test")
File "/home/usr/.local/lib/python3.9/site-packages/prefect/core/flow.py", line 1726, in register
registered_flow = client.register(
File "/home/usr/.local/lib/python3.9/site-packages/prefect/client/client.py", line 848, in register
serialized_flow = flow.serialize(build=build) # type: Any
File "/home/usr/.local/lib/python3.9/site-packages/prefect/core/flow.py", line 1507, in serialize
self.storage.add_flow(self)
File "/home/usr/.local/lib/python3.9/site-packages/prefect/storage/local.py", line 143, in add_flow
f.write(flow_to_bytes_pickle(flow))
File "/home/usr/.local/lib/python3.9/site-packages/prefect/utilities/storage.py", line 178, in flow_to_bytes_pickle
cloudpickle.dumps(flow, protocol=4), newline=False
File "/home/usr/.local/lib/python3.9/site-packages/cloudpickle/cloudpickle_fast.py", line 73, in dumps
cp.dump(obj)
File "/home/usr/.local/lib/python3.9/site-packages/cloudpickle/cloudpickle_fast.py", line 633, in dump
return Pickler.dump(self, obj)
TypeError: cannot pickle 'weakref' object
It only happens when I try to flow.register(project="QA-test").
If I only add flow.run() it works fine.
This is the beggining of the code:
#task(name="T3")
def feed_validator():
jsonpath = ''
results = []
urlsData = load_data('urls2.json')
for feed in urlsData:
for country in feed["countries"]:
feedUrl = feed['url']
if feed['type'] == 'Availability':
feedUrl = feedUrl.replace('/us/', "/" + country.lower() + "/")
feedData = getFeed(feedUrl)
feedDateInfo = isFeedUpdated(feedData)
if feed['network'] == 'network-samp(gate)':
feed['network'] = 'network-samp'
elif feed['network'] == 'network-samp(samp)':
feed['network'] = 'network-samp2'
country = ''
elif feed['network'] == 'network-samp3(samp3)':
feed['network'] = 'network-samp3'
country = ''
elif feed['type'] == 'Catalog':
country = ''
results.append((feed['platform'], feed['type'], feed['network'], country, feedDateInfo[1]))
orderedResults = multisort(list(results), ((4, False), (1, False), (2, False), (3, False)))
report = email_sender.createResultsEmail(orderedResults)
email_sender.sendEmail(report.as_string())
with Flow("test", storage=Local()) as flow:
feed_validator()
flow.register(project_name="QA-Test")
flow.run()

When using Local storage, Prefect by default uses pickled flow. You can switch to using script storage, as described on this Documentation page

!!python/object:__main__.ClassName throwing exception

I have a YAML file: ./YAML/simpleData.yml
- name: 'Somu'
age: 26
content:
- name: 'Neo'
age: 27
content: []
- name: 'Ari'
age: 26
content: []
And I'm trying to parse it using PyYAML via:
import yaml
# Creating objects directly with the YAML module:
print("Attempting Direct Object Load: ")
class Person:
def __init__(self, name, age, con):
self.name = name
self.age = hp
self.content = con
def __repr__(self):
return "%s(name=%r, hp=%r, sp=%r)" % (
self.__class__.__name__, self.name, self.age, self.content)
def printData(self):
print(self.name)
print(self.age)
if self.content:
for per in self.content:
print("-->", end="")
per.printData()
# Data load:
person_obj = None
data = ""
try:
with open('YAML/simpleData.yml') as source:
for line in source:
data += line
except Exception as err:
print("An exception occurred: " + str(err))
person_obj = yaml.load("""!!python/object:__main__.Person\n""" + data)
if not person_obj:
print("Data Loading Failed..! EXITING!!")
exit(1)
person_obj.printData()
I'm new to Python, and thus can't determine what I'm doing wrong, due to which this exception is being raised:
yaml.constructor.ConstructorError: expected a mapping node, but found sequence
in "<unicode string>", line 1, column 1:
!!python/object:__main__.Person
^
How do I fix this?
Full Output Dump:
Attempting Direct Object Load:
Traceback (most recent call last):
File "/home/somu/Programming/python/HeadFirstPython/yamlIntro.py", line 106, in <module>
person_obj = yaml.load("""!!python/object:__main__.Person\n""" + data)
File "/home/somu/Programming/python/HeadFirstPython/venv/lib/python3.6/site-packages/yaml/__init__.py", line 72, in load
return loader.get_single_data()
File "/home/somu/Programming/python/HeadFirstPython/venv/lib/python3.6/site-packages/yaml/constructor.py", line 37, in get_single_data
return self.construct_document(node)
File "/home/somu/Programming/python/HeadFirstPython/venv/lib/python3.6/site-packages/yaml/constructor.py", line 46, in construct_document
for dummy in generator:
File "/home/somu/Programming/python/HeadFirstPython/venv/lib/python3.6/site-packages/yaml/constructor.py", line 578, in construct_python_object
state = self.construct_mapping(node, deep=deep)
File "/home/somu/Programming/python/HeadFirstPython/venv/lib/python3.6/site-packages/yaml/constructor.py", line 204, in construct_mapping
return super().construct_mapping(node, deep=deep)
File "/home/somu/Programming/python/HeadFirstPython/venv/lib/python3.6/site-packages/yaml/constructor.py", line 122, in construct_mapping
node.start_mark)
yaml.constructor.ConstructorError: expected a mapping node, but found sequence
in "<unicode string>", line 1, column 1:
!!python/object:__main__.Person
^
Process finished with exit code 1

At the root, c.q. top level, of your file you have a sequence. The first element of which is a mapping with among others the key-value pair name: Somu.
If you want to load this using PyYAML in the way you described, you should strip of the first two characters of each line:
data += line[2:]
or insert the !!python/object:__main__.Person after the first dash:
data = data.replace('- ', '- !!python/object:__main__.Person\n', 1)

to get android manifest meta data value by python xpath

<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.xiaomi.oga">
...
<meta-data
android:name="xxx"
android:value="xxx" >
</meta-data>
<meta-data
android:name="OTA_TYPE"
android:value="daily_build_test" />
</application>
</manifest>
I want get the daily_build_test above by python
So far, I tried:
import lxml.etree as ET
import os
ns='{http://schemas.android.com/apk/res/android}'
nametag=ns+'name'
categorytag='category'
packagetag='package'
class XmlParser:
def __init__(self, file):
self.file = file
self.tree = None
self.root = None
def __parse(self):
if self.tree is None or self.root is None:
parser = ET.XMLParser(strip_cdata=False)
self.tree = ET.parse(self.file, parser)
self.root = self.tree.getroot()
def gettree(self):
self.__parse()
return self.tree
def getroot(self):
self.__parse()
return self.root
def get_meta_data(parser):
tree = parser.gettree()
#value = tree.xpath('/manifest/application//meta-data[#*=\'OTA_TYPE\']/#*[2]')
NS = {'android' : 'http://schemas.android.com/apk/res/android'}
value = tree.xpath('/manifest/application/meta-data[#a:name=\'OTA_TYPE\']/#a:value', namespace=NS)
print '%s' %(value)
return value
if __name__ == '__main__':
file = os.environ['PYTHON_ARG']
parser = XmlParser(file)
meta_data = get_meta_data(parser)
print '%s' %(meta_data)
and I got :
Traceback (most recent call last):
File "<stdin>", line 44, in <module>
File "<stdin>", line 37, in get_meta_data
File "src/lxml/lxml.etree.pyx", line 2272, in lxml.etree._ElementTree.xpath (src/lxml/lxml.etree.c:70786)
File "src/lxml/xpath.pxi", line 352, in lxml.etree.XPathDocumentEvaluator.__call__ (src/lxml/lxml.etree.c:179055)
File "src/lxml/xpath.pxi", line 95, in lxml.etree._XPathContext.registerVariables (src/lxml/lxml.etree.c:175335)
File "src/lxml/extensions.pxi", line 614, in lxml.etree._wrapXPathObject (src/lxml/lxml.etree.c:171074)
lxml.etree.XPathResultError: Unknown return type: dict
or ['daily_build_test']
when the commentted line :
value = tree.xpath('/manifest/application//meta-data[#*=\'OTA_TYPE\']/#*[2]')
is applied
I don't want the brackets the quotes.

The argument to tree.xpath is namespaces, not namespace. So:
NS = {'a' : 'http://schemas.android.com/apk/res/android'}
value = tree.xpath('/manifest/application/'
'meta-data[#a:name=\'OTA_TYPE\']/#a:value',
namespaces=NS)
Also note that you were using the key android in your NS dictionary, but a: in your query. The example above corrects the dictionary so that it matches the prefix you're using.

Thanks to larsks, I finally get I want by:*
def get_meta_data(parser):
tree = parser.gettree()
NS = {'a' : 'http://schemas.android.com/apk/res/android'}
value = tree.xpath('/manifest/application/meta-data[#a:name=\'OTA_TYPE\']/#a:value', namespaces=NS)
print '%s' %(value[0])
# NOTE value is a list of length 0.
return value[0]

Too many indices error coming up in Python

This is my first bash at using python, I am making an array of fathers and children, I have a dataset in a .csv file which I need to go into python so I can later take it over to java script. However the same error message keeps coming which I have put below. Further down is my script. Would be most grateful for any advice!
Gabriella
>>> runfile('/Users/gkountourides/Desktop/FunctionalExperiment/untitled0.py', wdir='/Users/gkountourides/Desktop/FunctionalExperiment')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "//anaconda/lib/python3.5/site-packages/spyderlib/widgets/externalshell/sitecustomize.py", line 714, in runfile
execfile(filename, namespace)
File "//anaconda/lib/python3.5/site-packages/spyderlib/widgets/externalshell/sitecustomize.py", line 89, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "/Users/gkountourides/Desktop/FunctionalExperiment/untitled0.py", line 41, in <module>
father,child = fc_csv_to_javascript_arrays("/Users/gkountourides/Desktop/FunctionalExperiment/fatherChild.csv")
File "/Users/gkountourides/Desktop/FunctionalExperiment/untitled0.py", line 38, in fc_csv_to_javascript_arrays
father_str, child_str = from_fc_array_to_fc_string(full_array)
File "/Users/gkountourides/Desktop/FunctionalExperiment/untitled0.py", line 30, in from_fc_array_to_fc_string
father_array = input_2d_array[:,0]
IndexError: too many indices for array
>>>
And then my actual script:
import glob
import numpy as np
def list_all_jpgs():
javascript_string = "["
for filename in glob.glob("*.jpg"):
javascript_string += '"' + filename + '",'
javascript_string = javascript_string[0:-1] + "]"
return javascript_string
def load_into_np_array(input_csv_file):
loaded_array = np.genfromtxt(input_csv_file, dtype=str, delimiter=",")
return loaded_array
def from_single_array_to_string(input_1d_array):
output_string = '['
for entry in input_1d_array:
output_string += '"'+str(entry)+'",'
output_string = output_string[0:-1]+']'
return output_string
def from_fc_array_to_fc_string(input_2d_array):
father_array = input_2d_array[:,0]
child_array = input_2d_array[:,1]
father_string = from_single_array_to_string(father_array)
child_string = from_single_array_to_string(child_array)
return father_string, child_string
def fc_csv_to_javascript_arrays(csv_input):
full_array = load_into_np_array(csv_input)
father_str, child_str = from_fc_array_to_fc_string(full_array)
return father_str, child_str
father,child = fc_csv_to_javascript_arrays("/Users/gkountourides/Desktop/FunctionalExperiment/fatherChild.csv")
print(father)
print(child)

The too many indices error indicates that input_2d_array is not a two 2d array. genfromtxt() is not returning what you are expecting.
numpy.genfromtxt produces array of what looks like tuples, not a 2D array—why?
http://docs.scipy.org/doc/numpy/reference/generated/numpy.genfromtxt.html

Django Error: TypeError: expected string or buffer

I'm writing code for a Django-based static blog, but I am coming across this similar issue across 3 or 4 different areas of my code. I figured if I can get one fixed then I can get the others fixed as well. My code of focus will be a django-command that I call update_blog1. Here's the traceback...
Traceback (most recent call last):
File "C:\Python34\lib\site-packages\django\core\handlers\base.py", line 132, in get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:/Users/Jaysp_000/firstSite/PROJECTone\blog_static\views.py", line 179, in archive
{'posts' : posts}
File "C:/Users/Jaysp_000/firstSite/PROJECTone\blog_static\views.py", line 14, in render_response
return render_to_response(*args, **kwargs)
File "C:\Python34\lib\site-packages\django\shortcuts.py", line 45, in render_to_response using=using)
File "C:\Python34\lib\site-packages\django\template\loader.py", line 116, in render_to_string
template_name, context, context_instance, dirs, dictionary)
File "C:\Python34\lib\site-packages\django\template\engine.py", line 221, in render_to_string
return t.render(context_instance)
File "C:\Python34\lib\site-packages\django\template\base.py", line 208, in render
with context.bind_template(self):
File "C:\Python34\lib\contextlib.py", line 59, in __enter__
return next(self.gen)
File "C:\Python34\lib\site-packages\django\template\context.py", line 235, in bind_template
updates.update(processor(self.request))
File "C:\Python34\lib\site-packages\django\template\context_processors.py", line 56, in i18n
context_extras['LANGUAGE_BIDI'] = translation.get_language_bidi()
File "C:\Python34\lib\site-packages\django\utils\translation\__init__.py", line 177, in get_language_bidi
return _trans.get_language_bidi()
File "C:\Python34\lib\site-packages\django\utils\translation\trans_real.py", line 263, in get_language_bidi
base_lang = get_language().split('-')[0]
AttributeError: 'NoneType' object has no attribute 'split'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Python34\lib\site-packages\django\utils\formats.py", line 103, in get_format
cached = _format_cache[cache_key]
KeyError: ('r', None)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Program Files (x86)\JetBrains\PyCharm 4.5.3\helpers\pycharm\django_manage.py", line 41, in <module>
run_module(manage_file, None, '__main__', True)
File "C:\Python34\lib\runpy.py", line 182, in run_module
return _run_module_code(code, init_globals, run_name, mod_spec)
File "C:\Python34\lib\runpy.py", line 96, in _run_module_code
mod_name, mod_spec, pkg_name, script_name)
File "C:\Python34\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:/Users/Jaysp_000/firstSite/PROJECTone\manage.py", line 10, in <module>
execute_from_command_line(sys.argv)
File "C:\Python34\lib\site-packages\django\core\management\__init__.py", line 338, in execute_from_command_line
utility.execute()
File "C:\Python34\lib\site-packages\django\core\management\__init__.py", line 330, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "C:\Python34\lib\site-packages\django\core\management\base.py", line 390, in run_from_argv
self.execute(*args, **cmd_options)
File "C:\Python34\lib\site-packages\django\core\management\base.py", line 441, in execute
output = self.handle(*args, **options)
File "C:/Users/Jaysp_000/firstSite/PROJECTone\blog_static\management\commands\update_blog1.py", line 78, in handle
resp = client.get(path)
File "C:\Python34\lib\site-packages\django\test\client.py", line 500, in get
**extra)
File "C:\Python34\lib\site-packages\django\test\client.py", line 303, in get
return self.generic('GET', path, secure=secure, **r)
File "C:\Python34\lib\site-packages\django\test\client.py", line 379, in generic
return self.request(**r)
File "C:\Python34\lib\site-packages\django\test\client.py", line 448, in request
response = self.handler(environ)
File "C:\Python34\lib\site-packages\django\test\client.py", line 122, in __call__
response = self.get_response(request)
File "C:\Python34\lib\site-packages\django\core\handlers\base.py", line 218, in get_response
response = self.handle_uncaught_exception(request, resolver, sys.exc_info())
File "C:\Python34\lib\site-packages\django\core\handlers\base.py", line 261, in handle_uncaught_exception
return debug.technical_500_response(request, *exc_info)
File "C:\Python34\lib\site-packages\django\views\debug.py", line 97, in technical_500_response
html = reporter.get_traceback_html()
File "C:\Python34\lib\site-packages\django\views\debug.py", line 384, in get_traceback_html
return t.render(c)
File "C:\Python34\lib\site-packages\django\template\base.py", line 209, in render
return self._render(context)
File "C:\Python34\lib\site-packages\django\template\base.py", line 201, in _render
return self.nodelist.render(context)
File "C:\Python34\lib\site-packages\django\template\base.py", line 903, in render
bit = self.render_node(node, context)
File "C:\Python34\lib\site-packages\django\template\debug.py", line 79, in render_node
return node.render(context)
File "C:\Python34\lib\site-packages\django\template\debug.py", line 89, in render
output = self.filter_expression.resolve(context)
File "C:\Python34\lib\site-packages\django\template\base.py", line 674, in resolve
new_obj = func(obj, *arg_vals)
File "C:\Python34\lib\site-packages\django\template\defaultfilters.py", line 771, in date
return formats.date_format(value, arg)
File "C:\Python34\lib\site-packages\django\utils\formats.py", line 136, in date_format
return dateformat.format(value, get_format(format or 'DATE_FORMAT', use_l10n=use_l10n))
File "C:\Python34\lib\site-packages\django\utils\formats.py", line 110, in get_format
for module in get_format_modules(lang):
File "C:\Python34\lib\site-packages\django\utils\formats.py", line 82, in get_format_modules
modules = _format_modules_cache.setdefault(lang, list(iter_format_modules(lang, settings.FORMAT_MODULE_PATH)))
File "C:\Python34\lib\site-packages\django\utils\formats.py", line 51, in iter_format_modules
if not check_for_language(lang):
File "C:\Python34\lib\site-packages\django\utils\translation\__init__.py", line 181, in check_for_language
return _trans.check_for_language(lang_code)
File "C:\Python34\lib\functools.py", line 472, in wrapper
result = user_function(*args, **kwds)
File "C:\Python34\lib\site-packages\django\utils\translation\trans_real.py", line 409, in check_for_language
if not language_code_re.search(lang_code):
TypeError: expected string or buffer
Here's my code for the update_blog1
from django.core.management.base import BaseCommand
from django.core.urlresolvers import reverse
from django.test.client import Client
import sys, os
from optparse import make_option
from P1config.settings import STATICBLOG_POST_DIRECTORY, STATICBLOG_COMPILE_DIRECTORY
class Command(BaseCommand):
help = "Compile blog posts from html to markdown, and upload images to S3 Defaults to processing only new blog posts"
option_list = BaseCommand.option_list + (
make_option(
'--all',
action='store_true',
dest='all',
default=False,
help='Get all blog posts, regardless of date'
),
make_option(
'--name',
action='store',
dest='post_name',
default=False,
help='Get named blog post'
),
)
def handle(self, *args, **options):
verbosity = int(options.get('verbosity'))
client = Client()
outdir = STATICBLOG_COMPILE_DIRECTORY
posts = []
previews = []
if options['all']:
if verbosity > 3:
print ('Compiling all blog posts')
posts = self._get_all_posts()
elif options['post_name']:
posts = self._get_named_posts(options['post_name'])
else:
if verbosity > 3:
print ('Compiling new blog posts')
posts = self._get_all_posts(new = True)
if verbosity > 3:
print ('%d posts found' % len(posts))
print ('----------------------------')
for post in posts:
if verbosity > 3:
print ("Compiling " + post['md_name'] + " to " + post['html_name'])
path = reverse('blog_static.views.archive') + post['path']
# path = '/preview/' + post['path']
resp = client.get(path)
if os.path.exists(outdir + post['path']) == False:
try:
with open(outdir + post['path'], 'r') as f:
pass
except IOError as e:
os.mkdir(outdir + post['path'])
with open(outdir + post['html_name'], 'wb') as f:
f.write(resp.content)
if len(posts) > 0 and verbosity > 3:
print ('----------------------------')
if verbosity > 3:
print ('Updating listings...')
print ('----------------------------')
path = reverse('blog_static.views.archive')
resp = client.get(path)
with open(STATICBLOG_COMPILE_DIRECTORY + 'index.html', 'wb') as f:
f.write(resp.content)
if verbosity > 3:
print ('Done')
def _get_all_posts(self, new = False):
posts = []
for item in os.listdir(STATICBLOG_POST_DIRECTORY):
post = self._create_post(item, new)
if post:
posts.append(post)
return posts
def _get_named_posts(self, post):
post_list = post.split(',')
posts = []
for item in post_list:
try:
with open(STATICBLOG_POST_DIRECTORY + item, 'r') as f:
post = self._create_post(item)
if post:
posts.append(post)
except IOError as e:
print ('\033[01;31m' + str(e) + '\033[0m', file= sys.stderr)
return posts
def _create_post(self, item, new = False):
outdir = STATICBLOG_POST_DIRECTORY
compiled_post = {
'md_name' : item,
'html_name' : '',
'path' : '',
'html' : '',
}
if item.endswith('.md'):
compiled_post['path'] = item.replace('.md', '')
compiled_post['html_name'] = compiled_post['path'] + '/index.html'
if new:
try:
with open(STATICBLOG_COMPILE_DIRECTORY + compiled_post['html_name'], 'r') as f:
return False
except IOError as e:
return compiled_post
else:
return compiled_post
If you look in the code, the methods _get_named_posts(), _create_post(), handle() each have a open() function embedded. Where these open() functions are (ex: open(outdir + post['path'], 'r' or open(STATICBLOG_POST_DIRECTORY + item, 'r')) is where the problem lies, as pointed out by PyCharm.
In my view.py file, I got have this...
# Django imports
from django.template import RequestContext
from django.shortcuts import render_to_response, render
from django.core.files.storage import get_storage_class
from django.core.files.base import ContentFile
from django.views.decorators.csrf import csrf_exempt
from django.db import models
# Create a 'shortcut' function to wrap request in RequestContext()
def render_response(req, *args, **kwargs):
"""Shortcut to wrap request in RequestContext"""
kwargs['context_instance'] = RequestContext(req)
return render_to_response(*args, **kwargs)
# Standard Python lib
import os, sys, urllib, hashlib
# 3rd party apps
import markdown
from markdown.inlinepatterns import ImagePattern, IMAGE_LINK_RE
# from config folder
from P1config.settings import STATICBLOG_COMPILE_DIRECTORY, \
STATICBLOG_POST_DIRECTORY, \
STATICBLOG_STORAGE
###################################################################################
class S3ImagePattern(ImagePattern):
""" Wrapper class to handle image matches in markdown document """
def handleMatch(self, match):
node = ImagePattern.handleMatch(self, match)
# check 'src' to ensure it is local
src = node.attrib.get('src')
storage_class = get_storage_class(STATICBLOG_STORAGE)
storage = storage_class()
# otherwise we need to do some downloading!
if 'http://' in src or 'https://' in src:
img_data = urllib.request.urlopen(src).read()
md5 = hashlib.md5()
md5.update(img_data)
name = md5.hexdigest() + '/' + os.path.basename(src)
else:
with open(STATICBLOG_POST_DIRECTORY + src) as fhandle:
img_data = fhandle.read()
name = src
print('Uploading ' + src, file=sys.stderr)
try:
storage.save(name, ContentFile(img_data))
node.attrib['src'] = storage.url(name)
print ('Uploaded ' + src + ' to ' + storage.url(name), file=sys.stderr)
except Exception as e:
print(str(e), file=sys.stderr)
print ('\033[01;31mUpload of %s failed\033[0m' % src, file=sys.stderr)
return node
def render_post(request, post_name):
""" Render a blog post based on a .post template
The used template is rendered as html in the folder defined
by STATICBLOG_COMPILE_DIRECTORY
"""
content = ""
mdown = markdown.Markdown(extensions = ['meta',])
mdown.inlinePatterns['image_link'] = S3ImagePattern(IMAGE_LINK_RE, mdown)
try:
post_file_dir = os.path.join(STATICBLOG_POST_DIRECTORY, post_name + '.md')
with open(post_file_dir, 'r') as pfDIR:
content = pfDIR.read() # opening and reading the ENTIRE '.md' document
html = mdown.convert(content) # converting file from '.md' to ".html"
except IOError as e:
print (str(e))
with open(os.path.join(STATICBLOG_POST_DIRECTORY, 'preview2.md')) as f:
content = f.read()
html = mdown.convert(content)
post = { 'content' : html, }
try:
post['date'] = mdown.Meta['date'][0]
post['title'] = mdown.Meta['title'][0]
post['author'] = mdown.Meta['author'][0]
post['summary'] = mdown.Meta['summary'][0]
post['tags'] = mdown.Meta['tags'][0]
except:
pass
meta = {}
if 'title' in post:
meta['title'] = post['title']
# Context Object containing the post, meta contexes
context = {'post' : post, 'meta' : meta}
return render_response( # but could I just use render?
request,
'post2.html',
context
)
def archive(request):
mdown = markdown.Markdown(extensions = ['meta',])
# Create an empty post list for now
posts = []
import string
# Look at every 'item' in the STATICBLOG_COMPILE_DIRECTORY
for item in os.listdir(STATICBLOG_COMPILE_DIRECTORY):
# if the 'item' in this directory ends with '.post' (like '.md' in a markdown file, or '.py' in a python file)
# More specifically, if there is a '.post' file located in this directory...
if item.endswith('.md'):
# ...continue on and...
continue
# ...attempt to use that 'item'
try:
# ...by opening it,
with open(os.path.join(STATICBLOG_POST_DIRECTORY, item + '.md')) as fhandle:
# ...reading the markdown file,
content = fhandle.read() # (opening and reading the ENTIRE '.md' document)
# ...and converting it to HTML.
mdown.convert(content) # (converting file from '.md' to ".html")
post = { 'name' : item, }
if 'title' in mdown.Meta and len(mdown.Meta['title'][0]) > 0:
# Add the markdown document's 'title' to post[]
# This stores the post's title from the "Meta" section of the '.md' document
post['title'] = mdown.Meta['title'][0]
# but if that doesnt work...
else:
# Add to the post list the item's Meta 'title', which simply takes \n
# the title of the .md document and removes the '-' from it, so that \n
# we can make it the post's title.
post['title'] = string.capwords(item.replace('-', ' '))
# ...and if there exists a 'date' in the item's meta attribute...
if 'date' in mdown.Meta:
# pass the 'date' info found in the Meta attribute to a 'date' \n
# variable created in the post list
post['date'] = mdown.Meta['date'][0]
posts.append(post)
except:
pass
from operator import itemgetter
posts = sorted(posts, key=itemgetter('date'))
posts.reverse()
return render_response( # but could I just use render?
request,
'archive.html',
{'posts' : posts}
)
#csrf_exempt
def handle_hook(request):
from django.http import HttpResponse
from django.core.management import call_command
result = call_command('update_blog', verbosity = 0)
return HttpResponse(result)
I dont know how to fix this. Can you let me know how to fix these issues? I don't know where I can put a string or buffer in my code.

Categories