We Keep Coding

Parallelizing the process

I want to parallelize the spec which is generated by _spectrum_generator. I am using futures.ThreadPoolExecutor which is called in _gather_lcms_data. The spec is passed through function. The file is in .mzML format. Below is the output that i get which is empty.
(base) ashish#14-ce3xxx:~/GNPS_LCMSDashboard$ python3 lcms_map.py
Empty DataFrame
Columns: [mz, rt, i, scan, index, polarity]
Index: []
The output should be look like this:
(base) ashish#14-ce3xxx:/media/ashish/ubuntu7/GNPS_LCMSDashboard$ python3 lcms_map.py
mz rt i scan index polarity
0 169.038696 0.003722 1652.959961 1 1 1
1 177.969086 0.003722 1786.755127 1 1 1
2 194.156967 0.003722 1802.361450 1 1 1
3 154.059418 0.003722 1840.889160 1 1 1
4 164.080978 0.003722 1973.758423 1 1 1
5 150.079514 0.003722 1976.528687 1 1 1
6 160.096634 0.003722 2057.728516 1 1 1
7 201.182205 0.003722 2077.768311 1 1 1
8 162.078735 0.003722 2101.843018 1 1 1
9 171.044205 0.003722 2223.230713 1 1 1
Below is the code of _spectrum_generator:
def _spectrum_generator(filename, min_rt, max_rt):
run = pymzml.run.Reader(filename, MS_precisions=MS_precisions)
# Don't do this if the min_rt and max_rt are not reasonable values
if min_rt <= 0 and max_rt > 1000:
for spec in run:
yield spec
else:
try:
min_rt_index = _find_lcms_rt(run, min_rt) # These are inclusive on left
max_rt_index = _find_lcms_rt(run, max_rt) + 1 # Exclusive on the right
for spec_index in tqdm(range(min_rt_index, max_rt_index)):
spec = run[spec_index]
yield spec
print("USED INDEX")
except:
run = pymzml.run.Reader(filename, MS_precisions=MS_precisions)
for spec in run:
yield spec
print("USED BRUTEFORCE")
Below is code of lcms_map.py:
import os
import pymzml
import numpy as np
import datashader as ds
from tqdm import tqdm
import json
import pandas as pd
import xarray
import time
import utils
import plotly.express as px
import plotly.graph_objects as go
from utils import _spectrum_generator
from utils import _get_scan_polarity
from multiprocessing import Pool
import concurrent.futures
from multiprocessing import Process
# Enum for polarity
POLARITY_POS = 1
POLARITY_NEG = 2
def _gather_lcms_data(filename, min_rt, max_rt, min_mz, max_mz, polarity_filter="None", top_spectrum_peaks=100, include_polarity=False):
all_mz = []
all_rt = []
all_polarity = []
all_i = []
all_scan = []
all_index = []
spectrum_index = 0
number_spectra = 0
all_msn_mz = []
all_msn_rt = []
all_msn_polarity = []
all_msn_scan = []
all_msn_level = []
#fun(filename, min_rt, max_rt)
for spec in _spectrum_generator(filename, min_rt, max_rt):
rt = spec.scan_time_in_minutes()
try:
# Still waiting for the window
if rt < min_rt:
continue
# pass
# We've passed the window
if rt > max_rt:
break
except:
pass
if polarity_filter == "None":
pass
else:
scan_polarity = _get_scan_polarity(spec)
if polarity_filter != scan_polarity:
continue
if spec.ms_level == 1:
spectrum_index += 1
number_spectra += 1
try:
# Filtering peaks by mz
if min_mz <= 0 and max_mz >= 2000:
peaks = spec.peaks("raw")
else:
peaks = spec.reduce(mz_range=(min_mz, max_mz))
# Filtering out zero rows
peaks = peaks[~np.any(peaks < 1.0, axis=1)]
# Sorting by intensity
peaks = peaks[peaks[:,1].argsort()]
peaks = peaks[-1 * top_spectrum_peaks:]
mz, intensity = zip(*peaks)
all_mz += list(mz)
all_i += list(intensity)
all_rt += len(mz) * [rt]
all_scan += len(mz) * [spec.ID]
all_index += len(mz) * [number_spectra]
# Adding polarity
if include_polarity is True:
scan_polarity = _get_scan_polarity(spec)
if scan_polarity == "Positive":
all_polarity += len(mz) * [POLARITY_POS]
else:
all_polarity += len(mz) * [POLARITY_NEG]
except:
pass
elif spec.ms_level > 1:
try:
msn_mz = spec.selected_precursors[0]["mz"]
if msn_mz < min_mz or msn_mz > max_mz:
continue
all_msn_mz.append(msn_mz)
all_msn_rt.append(rt)
all_msn_scan.append(spec.ID)
all_msn_level.append(spec.ms_level)
# Adding polarity
if include_polarity is True:
scan_polarity = _get_scan_polarity(spec)
if scan_polarity == "Positive":
all_msn_polarity.append(POLARITY_POS)
else:
all_msn_polarity.append(POLARITY_NEG)
except:
pass
ms1_results = {}
ms1_results["mz"] = all_mz
ms1_results["rt"] = all_rt
ms1_results["i"] = all_i
ms1_results["scan"] = all_scan
ms1_results["index"] = all_index
msn_results = {}
msn_results["precursor_mz"] = all_msn_mz
msn_results["rt"] = all_msn_rt
msn_results["scan"] = all_msn_scan
msn_results["level"] = all_msn_level
# Adding polarity
if include_polarity is True:
ms1_results["polarity"] = all_polarity
msn_results["polarity"] = all_msn_polarity
ms1_results = pd.DataFrame(ms1_results)
msn_results = pd.DataFrame(msn_results)
return ms1_results, number_spectra, msn_results
def _get_feather_filenames(filename):
output_ms1_filename = filename + ".ms1.feather"
output_msn_filename = filename + ".msn.feather"
return output_ms1_filename, output_msn_filename
# These are caching layers for fast loading
def _save_lcms_data_feather(filename):
output_ms1_filename, output_msn_filename = _get_feather_filenames(filename)
start=time.time()
# with Pool(5) as p:
# #ms1_results, number_spectra, msn_results = p.starmap(_gather_lcms_data, (filename, 0, 1000000, 0, 10000, "None", 100000, True))
# ms1_results, number_spectra, msn_results = _gather_lcms_data(filename, 0, 1000000, 0, 10000, polarity_filter="None", top_spectrum_peaks=100000, include_polarity=True)
# with concurrent.futures.ProcessPoolExecutor(max_workers=100) as executor:
# f=executor.submit(_gather_lcms_data, filename, 0, 1000000, 0, 10000, polarity_filter="None", top_spectrum_peaks=100000, include_polarity=True)
# ms1_results, number_spectra, msn_results = f.result()
ms1_results, number_spectra, msn_results = _gather_lcms_data(filename, 0, 1000000, 0, 10000, polarity_filter="None", top_spectrum_peaks=100000, include_polarity=True)
print(ms1_results.head(10))
print("Gathered data in", time.time() - start)
ms1_results = ms1_results.sort_values(by='i', ascending=False).reset_index()
ms1_results.to_feather(output_ms1_filename)
msn_results.to_feather(output_msn_filename)
_save_lcms_data_feather("/media/ashish/ubuntu7/GNPS_LCMSDashboard/QC_0.mzML")
How do i get the desired output by parallelizing. Suggest the changes that i need make in order to make it work.

As Simon Lundberg pointed out, you posted very complicated code, which makes it difficult to parallelize and even more difficult to explain how it is to be done. But if you were to present a simplified version of your code that was readily parallelizable, any answer would not be dealing with the complexities of your actual current code and would therefore be of little help. So I will try to create code that is an abstraction of your code's structure and then show how I would parallelize that. I am afraid that since you are not that familiar with multiprocessing, this may be rather difficult for you to follow.
First, a few observations about your code:
_gather_lcms_data currently is passed a filename and then using a generator function, _spectrum_generator, loops on all of its elements, called variable spec. In each loop iteration variable results are appended to various lists and variable number_spectra is optionally incremented. You have another variable spectrum_index that is also optionally incremented but its value is not otherwise used and could be eliminated. Finally, these lists are added to various dictionaries.
To parallelize the _gather_lcms_data function, it needs to process a single element, spec from the _spectrum_generator function so that we can run multiple invocations of this function in parallel. Consequently it needs to return a tuple of elements back to the main process which will do the necessary appending to lists and then create the dictionaries.
In your current code for each iteration of spec you optionally increment number_spectra and optionally append elements to various lists. Since we are now going to be parallelizing this function by returning individual elements, the main process must (1) accumulate the returned number_spectra value and optionally append the returned elements to result lists. Where in the original code you were not appending an element to a list for a given iteration, in the parallelized code you must return a None value so that the main process knows that for that iteration nothing needs to be appended.
In this abstraction, I have also reduced the number of lists down to two and I am generating dummy results.
First an abstraction of your current code.
def _spectrum_generator(filename, min_rt, max_rt):
#run = pymzml.run.Reader(filename, MS_precisions=MS_precisions)
run = [1, 2, 3, 4, 5, 6]
for spec in run:
yield spec
def _gather_lcms_data(filename, min_rt, max_rt, min_mz, max_mz, polarity_filter="None", top_spectrum_peaks=100, include_polarity=False):
# Remainder of the list declarations omitted for simplicity
all_mz = []
all_msn_mz= []
number_spectra = 0
for spec in _spectrum_generator(filename, min_rt, max_rt):
... # Code omitted for simplicity
number_spectra += 1 # Conditionally done
msn_mz = spec # Conditionally done
all_msn_mz.append(msn_mz)
mz = (spec * spec,) # Conditionally done
all_mz += list(mz)
...
ms1_results = {}
msn_results = {}
...
ms1_results["mz"] = all_mz
msn_results["precursor_mz"] = all_msn_mz
...
# Return
return ms1_results, number_spectra, msn_results
def _save_lcms_data_feather(filename):
ms1_results, number_spectra, msn_results = _gather_lcms_data(filename, 0, 1000000, 0, 10000, polarity_filter="None", top_spectrum_peaks=100000, include_polarity=True)
print(ms1_results)
print(number_spectra)
print(msn_results)
if __name__ == '__main__':
_save_lcms_data_feather("/media/ashish/ubuntu7/GNPS_LCMSDashboard/QC_0.mzML")
Prints:
{'mz': [1, 4, 9, 16, 25, 36]}
6
{'precursor_mz': [1, 2, 3, 4, 5, 6]}
This is the parallelized version of the above code:
def _spectrum_generator(filename, min_rt, max_rt):
#run = pymzml.run.Reader(filename, MS_precisions=MS_precisions)
run = [1, 2, 3, 4, 5, 6]
for spec in run:
yield spec
def _gather_lcms_data(spec, min_rt, max_rt, min_mz, max_mz, polarity_filter="None", top_spectrum_peaks=100, include_polarity=False):
# Remainder of the list declarations omitted for simplicity
number_spectra = 0
... # Code omitted for simplicity
number_spectra += 1
msn_mz = spec # Conditionally done. If not done then set msn_mz to None
mz = list((spec * spec,)) # Conditionally done. If not done then set mz to None
...
return mz, number_spectra, msn_mz
def _save_lcms_data_feather(filename):
from multiprocessing import Pool
from functools import partial
min_rt = 0
max_rt = 1000000
worker_function = partial(_gather_lcms_data, min_rt=min_rt, max_rt=max_rt, min_mz=0, max_mz=10000, polarity_filter="None", top_spectrum_peaks=1000000, include_polarity=True)
with Pool() as pool:
all_mz = []
all_msn_mz = []
number_spectra = 0
for mz, _number_spectra, msn_mz in pool.map(worker_function, _spectrum_generator(filename, min_rt, max_rt)):
if mz is not None:
all_mz += mz
number_spectra += _number_spectra
if msn_mz is not None:
all_msn_mz.append(msn_mz)
ms1_results = {}
msn_results = {}
ms1_results["mz"] = all_mz
msn_results["precursor_mz"] = all_msn_mz
print(ms1_results)
print(number_spectra)
print(msn_results)
if __name__ == '__main__':
_save_lcms_data_feather("/media/ashish/ubuntu7/GNPS_LCMSDashboard/QC_0.mzML")

How to scroll text in terminal window with Python/Curses? [duplicate]

In my Python script which uses Curses, I have a subwin to which some text is assigned. Because the text length may be longer than the window size, the text should be scrollable.
It doesn't seem that there is any CSS-"overflow" like attribute for Curses windows. The Python/Curses docs are also rather cryptic on this aspect.
Does anybody here have an idea how I can code a scrollable Curses subwindow using Python and actually scroll through it?
\edit: more precise question

OK with window.scroll it was too complicated to move the content of the window. Instead, curses.newpad did it for me.
Create a pad:
mypad = curses.newpad(40,60)
mypad_pos = 0
mypad.refresh(mypad_pos, 0, 5, 5, 10, 60)
Then you can scroll by increasing/decreasing mypad_pos depending on the input from window.getch() in cmd:
if cmd == curses.KEY_DOWN:
mypad_pos += 1
mypad.refresh(mypad_pos, 0, 5, 5, 10, 60)
elif cmd == curses.KEY_UP:
mypad_pos -= 1
mypad.refresh(mypad_pos, 0, 5, 5, 10, 60)

Right, I was a bit confused on how to utilize pads (in order to scroll text), and still couldn't figure it out after reading this post; especially since I wanted to use it in a context of the content being an existing "array of lines". So I prepared a small example, that shows similarities (and differences) between newpad and subpad:
#!/usr/bin/env python2.7
import curses
# content - array of lines (list)
mylines = ["Line {0} ".format(id)*3 for id in range(1,11)]
import pprint
pprint.pprint(mylines)
def main(stdscr):
hlines = begin_y = begin_x = 5 ; wcols = 10
# calculate total content size
padhlines = len(mylines)
padwcols = 0
for line in mylines:
if len(line) > padwcols: padwcols = len(line)
padhlines += 2 ; padwcols += 2 # allow border
stdscr.addstr("padhlines "+str(padhlines)+" padwcols "+str(padwcols)+"; ")
# both newpad and subpad are <class '_curses.curses window'>:
mypadn = curses.newpad(padhlines, padwcols)
mypads = stdscr.subpad(padhlines, padwcols, begin_y, begin_x+padwcols+4)
stdscr.addstr(str(type(mypadn))+" "+str(type(mypads)) + "\n")
mypadn.scrollok(1)
mypadn.idlok(1)
mypads.scrollok(1)
mypads.idlok(1)
mypadn.border(0) # first ...
mypads.border(0) # ... border
for line in mylines:
mypadn.addstr(padhlines-1,1, line)
mypadn.scroll(1)
mypads.addstr(padhlines-1,1, line)
mypads.scroll(1)
mypadn.border(0) # second ...
mypads.border(0) # ... border
# refresh parent first, to render the texts on top
#~ stdscr.refresh()
# refresh the pads next
mypadn.refresh(0,0, begin_y,begin_x, begin_y+hlines, begin_x+padwcols)
mypads.refresh()
mypads.touchwin()
mypadn.touchwin()
stdscr.touchwin() # no real effect here
#stdscr.refresh() # not here! overwrites newpad!
mypadn.getch()
# even THIS command erases newpad!
# (unless stdscr.refresh() previously):
stdscr.getch()
curses.wrapper(main)
When you run this, at first you will get something like (newpad left, subpad right):
┌────────────────────────┐ ┌────────────────────────┐
│Line 1 Line 1 Line 1 ───│ │Line 1 Line 1 Line 1 ───│
│Line 2 Line 2 Line 2 │ │Line 2 Line 2 Line 2 │
│Line 3 Line 3 Line 3 │ │Line 3 Line 3 Line 3 │
│Line 4 Line 4 Line 4 │ │Line 4 Line 4 Line 4 │
│Line 5 Line 5 Line 5 │ │Line 5 Line 5 Line 5 │
│Line 6 Line 6 Line 6 │
│Line 7 Line 7 Line 7 │
│Line 8 Line 8 Line 8 │
│Line 9 Line 9 Line 9 │
│Line 10 Line 10 Line 10 │
└────────────────────────┘
Some notes:
Both newpad and subpad should have their width/height sized to the content (num lines/max line width of the array of lines) + eventual border space
In both cases, you could allow extra lines with scrollok() - but not extra width
In both cases, you basically "push" a line at the bottom of the pad; and then scroll() up to make room for the next
The special refresh method that newpad has, then allows for just a region of this "whole content" to be shown on screen; subpad more-less has to be shown in the size it was instantiated in
If you draw the borders of the pads before adding content strings - then the borders will scroll up too (that is the ─── piece shown at the ...Line 1 ───│ part).
Useful links:
(n)curses pad in python not working
http://threeseas.net/vic/vic/terminal.py
How do I delete a curse window in python and restore background window?

Set the window.scrollok(True).
Documentation

I wanted to use a scrolling pad to display content of some large text files but this didn't work well because texts can have line breaks and it was pretty hard to figure out how many characters to display at a time to fit the good number of columns and rows.
So I decided to first split my text files in lines of exactly COLUMNS characters, padding with spaces when lines were too short. Then scrolling the text become more easy.
Here is a sample code to display any text file:
#!/usr/bin/python
# -*- coding: utf-8 -*-
import curses
import locale
import sys
def main(filename, filecontent, encoding="utf-8"):
try:
stdscr = curses.initscr()
curses.noecho()
curses.cbreak()
curses.curs_set(0)
stdscr.keypad(1)
rows, columns = stdscr.getmaxyx()
stdscr.border()
bottom_menu = u"(↓) Next line | (↑) Previous line | (→) Next page | (←) Previous page | (q) Quit".encode(encoding).center(columns - 4)
stdscr.addstr(rows - 1, 2, bottom_menu, curses.A_REVERSE)
out = stdscr.subwin(rows - 2, columns - 2, 1, 1)
out_rows, out_columns = out.getmaxyx()
out_rows -= 1
lines = map(lambda x: x + " " * (out_columns - len(x)), reduce(lambda x, y: x + y, [[x[i:i+out_columns] for i in xrange(0, len(x), out_columns)] for x in filecontent.expandtabs(4).splitlines()]))
stdscr.refresh()
line = 0
while 1:
top_menu = (u"Lines %d to %d of %d of %s" % (line + 1, min(len(lines), line + out_rows), len(lines), filename)).encode(encoding).center(columns - 4)
stdscr.addstr(0, 2, top_menu, curses.A_REVERSE)
out.addstr(0, 0, "".join(lines[line:line+out_rows]))
stdscr.refresh()
out.refresh()
c = stdscr.getch()
if c == ord("q"):
break
elif c == curses.KEY_DOWN:
if len(lines) - line > out_rows:
line += 1
elif c == curses.KEY_UP:
if line > 0:
line -= 1
elif c == curses.KEY_RIGHT:
if len(lines) - line >= 2 * out_rows:
line += out_rows
elif c == curses.KEY_LEFT:
if line >= out_rows:
line -= out_rows
finally:
curses.nocbreak(); stdscr.keypad(0); curses.echo(); curses.curs_set(1)
curses.endwin()
if __name__ == '__main__':
locale.setlocale(locale.LC_ALL, '')
encoding = locale.getpreferredencoding()
try:
filename = sys.argv[1]
except:
print "Usage: python %s FILENAME" % __file__
else:
try:
with open(filename) as f:
filecontent = f.read()
except:
print "Unable to open file %s" % filename
else:
main(filename, filecontent, encoding)
The main trick is the line:
lines = map(lambda x: x + " " * (out_columns - len(x)), reduce(lambda x, y: x + y, [[x[i:i+out_columns] for i in xrange(0, len(x), out_columns)] for x in filecontent.expandtabs(4).splitlines()]))
First, tabulations in the text are converted to spaces, then I used splitlines() method to convert my text in array of lines.
But some lines may be longer than our COLUMNS number, so I splitted each line in chunk of COLUMNS characters and then used reduce to transform the resulting list in a list of lines.
Finally, I used map to pad each line with trailing spaces so that its length is exactly COLUMNS characters.
Hope this helps.

This is the answer of this question:
How to make a scrolling menu in python-curses
This code allows you to create a little scrolling menu in a box from a list of strings.
You can also use this code getting the list of strings from a sqlite query or from a csv file.
To edit the max number of rows of the menu you just have to edit max_row.
If you press enter the program will print the selected string value and its position.
from __future__ import division #You don't need this in Python3
import curses
from math import *
screen = curses.initscr()
curses.noecho()
curses.cbreak()
curses.start_color()
screen.keypad( 1 )
curses.init_pair(1,curses.COLOR_BLACK, curses.COLOR_CYAN)
highlightText = curses.color_pair( 1 )
normalText = curses.A_NORMAL
screen.border( 0 )
curses.curs_set( 0 )
max_row = 10 #max number of rows
box = curses.newwin( max_row + 2, 64, 1, 1 )
box.box()
strings = [ "a", "b", "c", "d", "e", "f", "g", "h", "i", "l", "m", "n" ] #list of strings
row_num = len( strings )
pages = int( ceil( row_num / max_row ) )
position = 1
page = 1
for i in range( 1, max_row + 1 ):
if row_num == 0:
box.addstr( 1, 1, "There aren't strings", highlightText )
else:
if (i == position):
box.addstr( i, 2, str( i ) + " - " + strings[ i - 1 ], highlightText )
else:
box.addstr( i, 2, str( i ) + " - " + strings[ i - 1 ], normalText )
if i == row_num:
break
screen.refresh()
box.refresh()
x = screen.getch()
while x != 27:
if x == curses.KEY_DOWN:
if page == 1:
if position < i:
position = position + 1
else:
if pages > 1:
page = page + 1
position = 1 + ( max_row * ( page - 1 ) )
elif page == pages:
if position < row_num:
position = position + 1
else:
if position < max_row + ( max_row * ( page - 1 ) ):
position = position + 1
else:
page = page + 1
position = 1 + ( max_row * ( page - 1 ) )
if x == curses.KEY_UP:
if page == 1:
if position > 1:
position = position - 1
else:
if position > ( 1 + ( max_row * ( page - 1 ) ) ):
position = position - 1
else:
page = page - 1
position = max_row + ( max_row * ( page - 1 ) )
if x == curses.KEY_LEFT:
if page > 1:
page = page - 1
position = 1 + ( max_row * ( page - 1 ) )
if x == curses.KEY_RIGHT:
if page < pages:
page = page + 1
position = ( 1 + ( max_row * ( page - 1 ) ) )
if x == ord( "\n" ) and row_num != 0:
screen.erase()
screen.border( 0 )
screen.addstr( 14, 3, "YOU HAVE PRESSED '" + strings[ position - 1 ] + "' ON POSITION " + str( position ) )
box.erase()
screen.border( 0 )
box.border( 0 )
for i in range( 1 + ( max_row * ( page - 1 ) ), max_row + 1 + ( max_row * ( page - 1 ) ) ):
if row_num == 0:
box.addstr( 1, 1, "There aren't strings", highlightText )
else:
if ( i + ( max_row * ( page - 1 ) ) == position + ( max_row * ( page - 1 ) ) ):
box.addstr( i - ( max_row * ( page - 1 ) ), 2, str( i ) + " - " + strings[ i - 1 ], highlightText )
else:
box.addstr( i - ( max_row * ( page - 1 ) ), 2, str( i ) + " - " + strings[ i - 1 ], normalText )
if i == row_num:
break
screen.refresh()
box.refresh()
x = screen.getch()
curses.endwin()
exit()

Another way is to print the visible items using for-loop on list with slice notation. That is, you choose a specific part of the list to display, then add or subtract an increment to change the range every time you press a key up or down.
like list[ y : y + coverage ] where y is the start and coverage determines how many items in the list you'd like to display.
from curses import wrapper
import curses
def main(stdscr):
mY = curses.LINES
win = curses.newwin(100,50,0,50)
win.keypad(True)
numbers = [n for n in range(0,1001)]
ylen = len(numbers)
iny = 0
border_y = mY-5
def scroll(window):
[window.addstr(y, 0, f'{b} \n') for y, b in enumerate(numbers[iny:iny+border_y])]
window.refresh()
scroll(win)
### KEY PRESS ###
while(True):
ch = win.getkey()
if ch == 'KEY_UP':
if(iny>0):
iny-=1
scroll(win)
elif ch == 'KEY_DOWN':
if(iny<ylen-border_y):
iny+=1
scroll(win)
elif ch == 'q':
break
wrapper(main)

Python Convert Inches and Feet into Inches

I am trying to convert arrays that contain both Inches and Feet into Inches. The feet are denoted with a single quote (') and inches are denoted with double quotes ("). The data comes in the following forms:
[8'x10']
[60\" x 72\"]
[5'x8',5x8,60\"x92\"]
[8'10\"x12']
What I want:
["96x120"]
["60x72"]
["60x96","60x96","60x96","60x92"]
["106x144"]
What I have:
def ft_inch(numbers):
if str(numbers).find("x") > -1:
numbers=numbers.replace('[','').replace('"','').replace('[','').replace(']','')
try:
nom = numbers.split("x")[0]
nom=nom.replace(r'\\|\"|\]|\[','')
nom_one = nom.split("'")[0]
nom_two = nom.split("'")[1]
den = numbers.split("x")[1]
den=den.replace(r'\\|\"|\[|\]','')
den_one = den.split("'")[0]
den_two = den.split("'")[1]
ft=int(nom_one)*12
inch=nom_two.replace(r'\"| |\\','')
try:
inch=int(inch)
except:
print('B')
tmp = int(ft)+int(inch)
fts=int(den_one)*12
inchs=den_two.replace(r'\"| |\\','')
try:
inchs=int(inchs)
except:
print('B')
tmp_two = int(fts)+int(inch)
return f'["{tmp}x{tmp_two}"]'
except:
return numbers
else:
return numbers
x="[5'1x8'1]"
ft_inch(x)
This works for a single array as long as it has both feet and inches but fails if its only feet [8'x8']. If anyone has a simpler solution please let me know

A regex-based approach:
import re
inputs = [["8'1x10'1"], ["60\" x 72\""], ["5'x8'", "5x8", "60\"x92\""], ["8'10\"x12'"]]
for inpt in inputs:
sub_output = []
for measurement in inpt:
m = re.match(r"(\d+['\"]?)(\d+['\"]?)?x(\d+['\"]?)(\d+['\"]?)?",
"".join(measurement.split()))
groups = [m.groups()[:2], m.groups()[2:]]
result_inches = [0, 0]
for i, group in enumerate(groups):
for raw_val in group:
if raw_val == None:
continue
if '"' in raw_val:
result_inches[i] += int(raw_val[:-1])
elif "'" in raw_val:
result_inches[i] += int(raw_val[:-1])*12
else:
result_inches[i] += int(raw_val)*12
sub_output.append(result_inches)
print([f"{x}x{y}" for x, y in sub_output])
Output:
['108x132']
['60x72']
['60x96', '60x96', '60x92']
['106x144']
I saw your edit and included the ["8'1x10'1"] case :)

I rewrote the entire thing, but this seems to work:
input_ = ["8'x10'", "60\" x 72\"", "5'x8'","5x8","60\"x92\"", "8'10\"x12'", "8'1x10'1\""]
inches_only = []
for s in input_:
s.replace(" ", "")
sides = s.split("x")
new_sides = []
for side in sides:
inches = 0
split1 = side.split("'")
if len(split1) > 1 or (len(split1) == 1 and not side.__contains__('"')):
inches = int(split1[0]) * 12
split2 = side.split('"')
if len(split2) > 1:
inches += int(split2[0].split("'")[-1])
elif len(split2) == 1 and len(split1) > 1 and len(split1[1]) > 0:
inches += int(split1[1])
new_sides.append(str(inches) + '"')
inches_only.append("x".join(new_sides))
print(inches_only)
Output:
['96"x120"', '60"x72"', '60"x96"', '60"x96"', '60"x92"', '106"x144"', '97"x121"']

Taking values from one file and after some calculation and a bit changes need to print into another file in python

below is c.txt
CO11 CSE C1 8
CO12 ETC C1 8
CO13 Electrical C2 12
CO14 Mech E 5
my program needs to print a course summary on screen and save that summary into a file
named cr.txt. Given the above c.txt, your program output should look like
below. The content of course_report.txt should also be the same, except the last line. Course
names in the second column use * to indicate a compulsory course and – to indicate an elective
course. The fourth column is the number of students enrolled in that course. The fifth column is the average score of the course.
CID Name Points. Enrollment. Average.
----------------------------------
CO11 * CSE 8 2 81
CO12 * ETC 8 10 71
CO13 * Electrical 12 8 61
CO14 - Mech 5 4 51
----------------------------------
poor-performing subject is CO14 with an average 51.
cr.txt generated!
below is what I've tried:
def read(self):
ctype = []
fi = open("c.txt", "r")
l = fi.readline()
while l != "":
fields = l.strip().split(" ")
self.c.append(fields)
l = fi.readline().strip()
f.close()
# print(f"{'CID'}{'Name':>20}{'Points.':>16}{'Enrollment.':>18}{'Average.':>10}")
# print("-" * 67, end="")
print()
for i in range(0, len(self.c)):
for j in range(len(self.c[i])):
obj = self.c[i][j]
print(obj.ljust(18), end="")
print()
print("-" * 67, end="")
print()

you can try use 'file.read' or 'file.readlines' after use 'open' function, if you choose 'file.readlines' you'll have to use 'for row in file.readlines()' look my example with 'file.read':
headers = ['CID', 'Name', 'Points.', 'Enrollment.', 'Average.']
compulsory_course = ['CO11', 'CO12', 'CO13']
elective_course = ['CO14']
count = 0
with open('c.txt', 'r') as file_c:
file_c.seek(0, 0)
file_string = file_c.read().replace('\n', ' ')
fields = file_string.split(' ')
with open('cr.txt', 'w') as file_cr:
for field in headers:
file_cr.write(f'{field} ')
file_cr.write('\n')
for v in fields:
if count == 4:
file_cr.write('\n')
count = 0
count += 1
if v in compulsory_course:
file_cr.write(f'{v} * ')
continue
elif v in elective_course:
file_cr.write(f'{v} - ')
continue
elif count == 3:
file_cr.write(f' ')
continue
file_cr.write(f'{v} ')

Read a binary file *.SRS (Solar Radio Spectrograph)

I wanna read a binary file (K7120127.SRS) with caractristhics detailed in the word file (Documentacion SRS DATA.doc) ,2.2. chapter, that adding in the next link
https://drive.google.com/folderview?id=0B_NlxFaQkpgHb00yTm5kU0MyaUU&usp=sharing
In the link is included a viewer of that data (Srsdisp.exe), but i wanna process this data not only view it, that's why I'd want to read it in Python.
I know plot using matplotlib, but work with binary files is new for me. I 'd wanna plot something like this (That plot was made using the viewer included in the link)

Try that.
from struct import unpack
# constants from the file spec
RECORD_SIZE=826
RECORD_HEADER_SIZE=24
RECORD_ARRAY_SIZE=401
# verbosity values
VERBOSITY_ALL = 2 # print warnings and errors
VERBOSITY_ERRORS = 1 # print errors
VERBOSITY_NONE = 0 # print nothing
class SRSRecord:
"""Holds one 826 byte SRS Record."""
_site_to_name = {
1: "Palehua",
2: "Holloman",
3: "Learmonth",
4: "San Vito",
# add new site names here ..
}
def __init__(self):
self.year = None
self.month = None
self.day = None
self.hour = None
self.minute = None
self.seconds = None
self.site_number = None
self.site_name = None
self.n_bands_per_record = None
self.a_start_freq = None
self.a_end_freq = None
self.a_num_bytes = None
self.a_analyser_reference_level = None
self.a_analyser_attenuation = None
self.b_start_freq = None
self.b_end_freq = None
self.b_num_bytes = None
self.b_analyser_reference_level = None
self.b_analyser_attenuation = None
# dictionary that maps frequency in mega hertz to level
self.a_values = {}
# dictionary that maps frequency in mega hertz to level
self.b_values = {}
return
def _parse_srs_file_header(self, header_bytes, verbosity = VERBOSITY_ALL):
fields = unpack(
# General header information
'>' # (data packed in big endian format)
'B' # 1 Year (last 2 digits) Byte integer (unsigned)
'B' # 2 Month number (1 to 12) "
'B' # 3 Day (1 to 31) "
'B' # 4 Hour (0 to 23 UT) "
'B' # 5 Minute (0 to 59) "
'B' # 6 Second at start of scan (0 to 59) "
'B' # 7 Site Number (0 to 255) "
'B' # 8 Number of bands in the record (2) "
# Band 1 (A-band) header information
'h' # 9,10 Start Frequency (MHz) Word integer (16 bits)
'H' # 11,12 End Frequency (MHz) "
'H' # 13,14 Number of bytes in data record (401) "
'B' # 15 Analyser reference level Byte integer
'B' # 16 Analyser attenuation (dB) "
# Band 2 (B-band) header information
# 17-24 As for band 1
'H' # 17,18 Start Frequency (MHz) Word integer (16 bits)
'H' # 19,20 End Frequency (MHz) "
'H' # 21,22 Number of bytes in data record (401) "
'B' # 23 Analyser reference level Byte integer
'B', # 24 Analyser attenuation (dB) "
header_bytes)
self.year = fields[0]
self.month = fields[1]
self.day = fields[2]
self.hour = fields[3]
self.minute = fields[4]
self.seconds = fields[5]
# read the site number and work out the site name
self.site_number = fields[6]
if self.site_number not in SRSRecord._site_to_name.keys():
# got an unknown site number.. complain a bit..
if verbosity >= VERBOSITY_ALL:
print("Unknown site number: %s" % self.site_number)
print("A list of known site numbers follows:")
for site_number, site_name in SRSRecord._site_to_name.items():
print("\t%s: %s" % (site_number, site_name))
# then set the site name to unknown.
self.site_name = "UnknownSite"
else:
# otherwise look up the site using our lookup table
self.site_name = SRSRecord._site_to_name[self.site_number]
# read the number of bands
self.n_bands_per_record = fields[7] # should be 2
if self.n_bands_per_record != 2 and verbosity >= VERBOSITY_ERRORS:
print("Warning.. record has %s bands, expecting 2!" % self.n_bands_per_record)
# read the a record meta data
self.a_start_freq = fields[8]
self.a_end_freq = fields[9]
self.a_num_bytes = fields[10]
if self.a_num_bytes != 401 and verbosity >= VERBOSITY_ERRORS:
print("Warning.. record has %s bytes in the a array, expecting 401!" %
self.a_num_bytes)
self.a_analyser_reference_level = fields[11]
self.a_analyser_attenuation = fields[12]
# read the b record meta data
self.b_start_freq = fields[13]
self.b_end_freq = fields[14]
self.b_num_bytes = fields[15]
if self.b_num_bytes != 401 and verbosity >= VERBOSITY_ERRORS:
print("Warning.. record has %s bytes in the b array, expecting 401!" %
self.b_num_bytes)
self.b_analyser_reference_level = fields[16]
self.b_analyser_attenuation = fields[17]
return
def _parse_srs_a_levels(self, a_bytes):
# unpack the frequency/levels from the first array
for i in range(401):
# freq equation from the srs file format spec
freq_a = 25 + 50 * i / 400.0
level_a = unpack('>B', a_bytes[i])[0]
self.a_values[freq_a] = level_a
return
def _parse_srs_b_levels(self, b_bytes):
for i in range(401):
# freq equation from the srs file format spec
freq_b = 75 + 105 * i / 400.0
level_b = unpack('>B', b_bytes[i])[0]
self.b_values[freq_b] = level_b
return
def __str__(self):
return ("%s/%s/%s, %s:%s:%s site: %s/%s bands: %s "
"[A %s->%s MHz ref_level: %s atten: %s dB], "
"[B %s->%s MHz ref_level: %s atten: %s dB]"
)% (
self.day, self.month, self.year,
self.hour, self.minute, self.seconds,
self.site_number, self.site_name,
self.n_bands_per_record,
self.a_start_freq, self.a_end_freq,
self.a_analyser_reference_level, self.a_analyser_attenuation,
self.b_start_freq, self.b_end_freq,
self.b_analyser_reference_level, self.b_analyser_attenuation,
)
def _dump(self, values):
freqs = values.keys()
freqs.sort()
for freq in freqs:
print "%5s %s" % (freq, values[freq])
return
def dump_a(self):
self._dump(self.a_values)
return
def dump_b(self):
self._dump(self.b_values)
return
def read_srs_file(fname):
"""Parses an srs file and returns a list of SRSRecords."""
# keep the records we read in here
srs_records = []
f = open(fname, "rb")
while True:
# read raw record data
record_data = f.read(RECORD_SIZE)
# if the length of the record data is zero we've reached the end of the data
if len(record_data) == 0:
break
# break up the record bytes into header, array a and array b bytes
header_bytes = record_data[:RECORD_HEADER_SIZE]
a_bytes = record_data[RECORD_HEADER_SIZE : RECORD_HEADER_SIZE + RECORD_ARRAY_SIZE]
b_bytes = record_data[RECORD_HEADER_SIZE + RECORD_ARRAY_SIZE :
RECORD_HEADER_SIZE + 2 * RECORD_ARRAY_SIZE]
# make a new srs record
record = SRSRecord()
record._parse_srs_file_header(header_bytes, verbosity = VERBOSITY_ERRORS)
record._parse_srs_a_levels(a_bytes)
record._parse_srs_b_levels(b_bytes)
srs_records.append(record)
return srs_records
if __name__ == "__main__":
# parse the file.. (this is where the magic happens ;)
srs_records = read_srs_file(fname = "K7120127.SRS")
# play with the data
for i in range(3):
print srs_records[i]
r0 = srs_records[0]
r0.dump_a()
r0.dump_b()