We Keep Coding

NotImplementedError: 'split_respect_sentence_boundary=True' is only compatible with split_by='word'

I have the following lines of code
from haystack.document_stores import InMemoryDocumentStore, SQLDocumentStore
from haystack.nodes import TextConverter, PDFToTextConverter,PreProcessor
from haystack.utils import clean_wiki_text, convert_files_to_docs, fetch_archive_from_http, print_answers
doc_dir = "C:\\Users\\abcd\\Downloads\\PDF Files\\"
docs = convert_files_to_docs(dir_path=doc_dir, clean_func=None, split_paragraphs=True
preprocessor = PreProcessor(
clean_empty_lines=True,
clean_whitespace=True,
clean_header_footer=True,
split_by="passage",
split_length=2)
doc = preprocessor.process(docs)
When i try to run it, i get the following error message
NotImplementedError Traceback (most recent call last)
c:\Users\abcd\Downloads\solr9.ipynb Cell 27 in <cell line: 23>()
16 print(type(docs))
17 preprocessor = PreProcessor(
18 clean_empty_lines=True,
19 clean_whitespace=True,
20 clean_header_footer=True,
21 split_by="passage",
22 split_length=2)
---> 23 doc = preprocessor.process(docs)
File ~\AppData\Roaming\Python\Python39\site-packages\haystack\nodes\preprocessor\preprocessor.py:167, in PreProcessor.process(self, documents, clean_whitespace, clean_header_footer, clean_empty_lines, remove_substrings, split_by, split_length, split_overlap, split_respect_sentence_boundary, id_hash_keys)
165 ret = self._process_single(document=documents, id_hash_keys=id_hash_keys, **kwargs) # type: ignore
166 elif isinstance(documents, list):
--> 167 ret = self._process_batch(documents=list(documents), id_hash_keys=id_hash_keys, **kwargs)
168 else:
169 raise Exception("documents provided to PreProcessor.prepreprocess() is not of type list nor Document")
File ~\AppData\Roaming\Python\Python39\site-packages\haystack\nodes\preprocessor\preprocessor.py:225, in PreProcessor._process_batch(self, documents, id_hash_keys, **kwargs)
222 def _process_batch(
223 self, documents: List[Union[dict, Document]], id_hash_keys: Optional[List[str]] = None, **kwargs
224 ) -> List[Document]:
--> 225 nested_docs = [
226 self._process_single(d, id_hash_keys=id_hash_keys, **kwargs)
...
--> 324 raise NotImplementedError("'split_respect_sentence_boundary=True' is only compatible with split_by='word'.")
326 if type(document.content) is not str:
327 logger.error("Document content is not of type str. Nothing to split.")
NotImplementedError: 'split_respect_sentence_boundary=True' is only compatible with split_by='word'.
I don't even have split_respect_sentence_boundary=True as my argument and also i don't have split_by='word' rather i have it set as split_by="passage".
This is the same error if i try changing it to split_by="sentence".
Do let me know if i am missing out anything here.
Tried using split_by="sentence" but getting same error.

As you can see in the PreProcessor API docs, the default value for split_respect_sentence_boundary is True.
In order to make your code work, you should specify split_respect_sentence_boundary=False:
preprocessor = PreProcessor(
clean_empty_lines=True,
clean_whitespace=True,
clean_header_footer=True,
split_by="passage",
split_length=2,
split_respect_sentence_boundary=False)
I agree that this behavior is not intuitive.
Currently, this node is undergoing a major refactoring.

Getting "AttributeError: 'NoneType' object has no attribute 'groupdict'" when executing an Excel file in Python using the Formulas library

My problem is to save data into an excel file and then make it run its calculations and then read it using python without having to open it. For this I have tried using the formulas library. It worked perfectly fine on a test excel sheet that I tries that had a few calculations. However when I try it on a larger excel sheet with a lot more and complex calculations I get this error message.
AttributeError: 'NoneType' object has no attribute 'groupdict
I will also post the entire error message here:
AttributeError: 'NoneType' object has no attribute 'groupdict'
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
/tmp/ipykernel_113/60233469.py in <module>
1 fpath = 'MTOOutput.xlsx'
2 dirname = 'MTOOutput'
----> 3 xl_model = formulas.ExcelModel().loads(fpath).finish(circular=True)
4 xl_model.calculate()
5 xl_model.write(dirpath=dirname)
~/.local/lib/python3.8/site-packages/formulas/excel/__init__.py in loads(self, *file_names)
93 def loads(self, *file_names):
94 for filename in file_names:
---> 95 self.load(filename)
96 return self
97
~/.local/lib/python3.8/site-packages/formulas/excel/__init__.py in load(self, filename)
98 def load(self, filename):
99 book, context = self.add_book(filename)
--> 100 self.pushes(*book.worksheets, context=context)
101 return self
102
~/.local/lib/python3.8/site-packages/formulas/excel/__init__.py in pushes(self, context, *worksheets)
106 def pushes(self, *worksheets, context=None):
107 for ws in worksheets:
--> 108 self.push(ws, context=context)
109 return self
110
~/.local/lib/python3.8/site-packages/formulas/excel/__init__.py in push(self, worksheet, context)
119 for c in row:
120 if hasattr(c, 'value'):
--> 121 self.add_cell(
122 c, context, references=references,
123 formula_references=formula_references,
~/.local/lib/python3.8/site-packages/formulas/excel/__init__.py in add_cell(self, cell, context, references, formula_references, formula_ranges, external_links)
232 val = cell.data_type == 'f' and val[:2] == '==' and val[1:] or val
233 check_formula = cell.data_type != 's'
--> 234 cell = Cell(crd, val, context=ctx, check_formula=check_formula).compile(
235 references=references, context=ctx
236 )
~/.local/lib/python3.8/site-packages/formulas/cell.py in compile(self, references, context)
88 def compile(self, references=None, context=None):
89 if self.builder:
---> 90 func = self.builder.compile(
91 references=references, context=context, **{CELL: self.range}
92 )
~/.local/lib/python3.8/site-packages/formulas/builder.py in compile(self, references, context, **inputs)
127 else:
128 try:
--> 129 i[k] = Ranges().push(k, context=context)
130 except ValueError:
131 i[k] = None
~/.local/lib/python3.8/site-packages/formulas/ranges.py in push(self, ref, value, context)
167
168 def push(self, ref, value=sh.EMPTY, context=None):
--> 169 rng = self.get_range(self.format_range, ref, context)
170 return self.set_value(rng, value)
171
~/.local/lib/python3.8/site-packages/formulas/ranges.py in get_range(format_range, ref, context)
159 def get_range(format_range, ref, context=None):
160 ctx = (context or {}).copy()
--> 161 for k, v in _re_range.match(ref).groupdict().items():
162 if v is not None:
163 if k == 'ref':
AttributeError: 'NoneType' object has no attribute 'groupdict'
I initially had not used the circular=True argument inside finish. But after a bit of trouble shooting I realized that could be the problem. I say that could because I did not create the excel sheet and hence do not know much about what is in there.
I also tried installing the formulas[all] version again after a bit of troubleshooting before getting the same error. I tried including the use of range references in my test file to check if that was causing the issue. However the test file worked fine with range references. I was wondering if anyone who had faced an issue like this with formulas had found a solution or an alternative.
PS: It would not be possible for me to attach the excel file as it is work related.
Thanks and Regards,
Yadhu

Backtrader 'File Not Found Error [ERRNO 2]'. Error with sample code from Backtrader Github page

I ran across this error when trying to run the backtrader sample setup code found on github. After doing some research I found that the error probably stems from the Yahoo Finance API being out of date or no longer compatible with the backtrader package. I plan on using online data feeds for the backtesting I want to do, so does anyone know how I can fix the yahoo data feed issue? Some online sources suggested digging into the source code. I already tried those suggestions with no avail.
from datetime import datetime
import backtrader as bt
class SmaCross(bt.SignalStrategy):
def __init__(self):
sma1, sma2 = bt.ind.SMA(period=10), bt.ind.SMA(period=30)
crossover = bt.ind.CrossOver(sma1, sma2)
self.signal_add(bt.SIGNAL_LONG, crossover)
cerebro = bt.Cerebro()
cerebro.addstrategy(SmaCross)
data0 = bt.feeds.YahooFinanceData(dataname='AAPL', fromdate=datetime(2011, 1, 1),
todate=datetime(2012, 12, 31))
cerebro.adddata(data0)
cerebro.run()
cerebro.plot()
Error Stack:
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
<ipython-input-2-4abfaa589128> in <module>
12 cerebro.adddata(data0)
13
---> 14 cerebro.run()
15 cerebro.plot()
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/cerebro.py in run(self, **kwargs)
1125 # let's skip process "spawning"
1126 for iterstrat in iterstrats:
-> 1127 runstrat = self.runstrategies(iterstrat)
1128 self.runstrats.append(runstrat)
1129 if self._dooptimize:
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/cerebro.py in runstrategies(self, iterstrat, predata)
1208 if self._exactbars < 1: # datas can be full length
1209 data.extend(size=self.params.lookahead)
-> 1210 data._start()
1211 if self._dopreload:
1212 data.preload()
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/feed.py in _start(self)
201
202 def _start(self):
--> 203 self.start()
204
205 if not self._started:
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/feeds/yahoo.py in start(self)
354
355 # Prepared a "path" file - CSV Parser can take over
--> 356 super(YahooFinanceData, self).start()
357
358
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/feeds/yahoo.py in start(self)
92
93 def start(self):
---> 94 super(YahooFinanceCSVData, self).start()
95
96 if not self.params.reverse:
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/feed.py in start(self)
672 else:
673 # Let an exception propagate to let the caller know
--> 674 self.f = io.open(self.p.dataname, 'r')
675
676 if self.p.headers:
FileNotFoundError: [Errno 2] No such file or directory: 'AAPL'

This was an issue that popped up this week. We have fixed it on Backtrader2 here. Backtrader2 was created to keep urgent bug fixes up to date, since the original backtrader is a closed repo now.
If you wish, there is only one line change. You can adjust your local backtrader file to match this: https://github.com/backtrader2/backtrader/pull/67/files
In file backtrader/feeds/yahoo.py, you will find the following code at line 269:
crumb = None
sess = requests.Session()
Add the following code on the next line:
sess.headers['User-Agent'] = 'backtrader'

Particle tracking with trackpy gives 'keframe' error?

I have tiff image stacks of 1000 frames in which I'm trying to track particles diffusing in Brownian motion. So I'm using trackpy for particle tracking: import the tiff stack, locate the features (particles) and then plot their trajectories. I'm using the following code (as from the trackpy walkthrough page):
import pims
import trackpy as tp
frames = pims.open("image.tif")
f = tp.locate(frames[0], 9, invert=False)
But this last line (tp.locate) gives a traceback:
AttributeError Traceback (most recent call last)
<ipython-input-78-0fbce96715a7> in <module>
----> 1 f = tp.locate(frames[0], 9, invert=False)
~\anaconda3\lib\site-packages\slicerator\__init__.py in __getitem__(self, i)
186 indices, new_length = key_to_indices(i, len(self))
187 if new_length is None:
--> 188 return self._get(indices)
189 else:
190 return cls(self, indices, new_length, propagate_attrs)
~\anaconda3\lib\site-packages\pims\base_frames.py in __getitem__(self, key)
96 """__getitem__ is handled by Slicerator. In all pims readers, the data
97 returning function is get_frame."""
---> 98 return self.get_frame(key)
99
100 def __iter__(self):
~\anaconda3\lib\site-packages\pims\tiff_stack.py in get_frame(self, j)
119 t = self._tiff[j]
120 data = t.asarray()
--> 121 return Frame(data, frame_no=j, metadata=self._read_metadata(t))
122
123 def _read_metadata(self, tiff):
~\anaconda3\lib\site-packages\pims\tiff_stack.py in _read_metadata(self, tiff)
124 """Read metadata for current frame and return as dict"""
125 # tags are only stored as a TiffTags object on the parent TiffPage now
--> 126 tags = tiff.keyframe.tags
127 md = {}
128 for name in ('ImageDescription', 'image_description'):
~\anaconda3\lib\site-packages\skimage\external\tifffile\tifffile.py in __getattr__(self, name)
2752 setattr(self, name, value)
2753 return value
-> 2754 raise AttributeError(name)
2755
2756 def __str__(self):
AttributeError: keyframe
where I'm going wrong?
I also tried using
imageio.imread("image.tif")
to import the image and then used
f=tp.locate(frames[0], 9, invert=False)
to locate the particles. But the output of this is supposed to be data for both x and y coordinates, like so
whereas what I get is just for the x axis:

I just had this same problem and solved it by entering "conda install pims" into my anaconda prompt. I had done "pip install pims" before, and I think that messed it up.

Using Sacred Module with iPython

I am trying to set up sacred for Python and I am going through the tutorial. I was able to set up sacred using pip install sacred with no issues. I am having trouble running the basic code:
from sacred import Experiment
ex = Experiment("hello_world")
Running this code returns the a ValueError:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-25-66f549cfb192> in <module>()
1 from sacred import Experiment
2
----> 3 ex = Experiment("hello_world")
/Users/ryandevera/anaconda/lib/python2.7/site-packages/sacred/experiment.pyc in __init__(self, name, ingredients)
42 super(Experiment, self).__init__(path=name,
43 ingredients=ingredients,
---> 44 _caller_globals=caller_globals)
45 self.default_command = ""
46 self.command(print_config, unobserved=True)
/Users/ryandevera/anaconda/lib/python2.7/site-packages/sacred/ingredient.pyc in __init__(self, path, ingredients, _caller_globals)
48 self.doc = _caller_globals.get('__doc__', "")
49 self.sources, self.dependencies = \
---> 50 gather_sources_and_dependencies(_caller_globals)
51
52 # =========================== Decorators ==================================
/Users/ryandevera/anaconda/lib/python2.7/site-packages/sacred/dependencies.pyc in gather_sources_and_dependencies(globs)
204 def gather_sources_and_dependencies(globs):
205 dependencies = set()
--> 206 main = Source.create(globs.get('__file__'))
207 sources = {main}
208 experiment_path = os.path.dirname(main.filename)
/Users/ryandevera/anaconda/lib/python2.7/site-packages/sacred/dependencies.pyc in create(filename)
61 if not filename or not os.path.exists(filename):
62 raise ValueError('invalid filename or file not found "{}"'
---> 63 .format(filename))
64
65 mainfile = get_py_file_if_possible(os.path.abspath(filename))
ValueError: invalid filename or file not found "None"
I am not sure why this error is returning. The documentation does not say anything about setting up an Experiment file prior to running the code. Any help would be greatly appreciated!

The traceback given indicates that the constructor for Experiment searches its namespace to find the file in which its defined.
Thus, to make the example work, place the example code into a file and run that file directly.
If you are using ipython, then you could always try using the %%python command, which will effectively capture the code you give it into a file before running it (in a separate python process).

According to the docs, if you're in IPython/Jupyter, you can allow the Experiment to run in a non-reproducible interactive environment:
ex = Experiment('jupyter_ex', interactive=True)
https://sacred.readthedocs.io/en/latest/experiment.html#run-the-experiment

The docs say it nicely (TL;DR: sacred checks this for you and fails in order to warn you)
Warning
By default, Sacred experiments will fail if run in an interactive
environment like a REPL or a Jupyter Notebook. This is an intended
security measure since in these environments reproducibility cannot be
ensured. If needed, this safeguard can be deactivated by passing
interactive=True to the experiment like this:
ex = Experiment('jupyter_ex', interactive=True)

Setting interactive=True doesn't work if you run the notebook as a script through ipython.
$ ipython code.ipynb
---------------------------------------------------------------------------
AssertionError Traceback (most recent call last)
Cell In[1], line 1
----> 1 ex = Experiment("image_classification", interactive=True)
2 ex.observers.append(NeptuneObserver(run=neptune_run))
File ~\miniconda3\envs\py38\lib\site-packages\sacred\experiment.py:119, in Experiment.__init__(self, name, ingredients, interactive, base_dir, additional_host_info, additional_cli_options, save_git_info)
117 elif name.endswith(".pyc"):
118 name = name[:-4]
--> 119 super().__init__(
120 path=name,
121 ingredients=ingredients,
122 interactive=interactive,
123 base_dir=base_dir,
124 _caller_globals=caller_globals,
125 save_git_info=save_git_info,
126 )
127 self.default_command = None
128 self.command(print_config, unobserved=True)
File ~\miniconda3\envs\py38\lib\site-packages\sacred\ingredient.py:75, in Ingredient.__init__(self, path, ingredients, interactive, _caller_globals, base_dir, save_git_info)
69 self.save_git_info = save_git_info
70 self.doc = _caller_globals.get("__doc__", "")
71 (
72 self.mainfile,
73 self.sources,
74 self.dependencies,
---> 75 ) = gather_sources_and_dependencies(
76 _caller_globals, save_git_info, self.base_dir
77 )
78 if self.mainfile is None and not interactive:
79 raise RuntimeError(
80 "Defining an experiment in interactive mode! "
81 "The sourcecode cannot be stored and the "
82 "experiment won't be reproducible. If you still"
83 " want to run it pass interactive=True"
84 )
File ~\miniconda3\envs\py38\lib\site-packages\sacred\dependencies.py:725, in gather_sources_and_dependencies(globs, save_git_info, base_dir)
723 def gather_sources_and_dependencies(globs, save_git_info, base_dir=None):
724 """Scan the given globals for modules and return them as dependencies."""
--> 725 experiment_path, main = get_main_file(globs, save_git_info)
727 base_dir = base_dir or experiment_path
729 gather_sources = source_discovery_strategies[SETTINGS["DISCOVER_SOURCES"]]
File ~\miniconda3\envs\py38\lib\site-packages\sacred\dependencies.py:596, in get_main_file(globs, save_git_info)
594 main = None
595 else:
--> 596 main = Source.create(globs.get("__file__"), save_git_info)
461 return Source(main_file, get_digest(main_file), repo, commit, is_dirty)
File ~\miniconda3\envs\py38\lib\site-packages\sacred\dependencies.py:382, in get_py_file_if_possible(pyc_name)
380 if pyc_name.endswith((".py", ".so", ".pyd")):
381 return pyc_name
--> 382 assert pyc_name.endswith(".pyc")
383 non_compiled_file = pyc_name[:-1]
384 if os.path.exists(non_compiled_file):
sacred==0.8.2