I have a class and in that class I have a method that calls multiple methods in it.
But the problem I am facing now is that when the method with the multiple methods in it duplicate parameter has.
And so when I am calling the method with the multiple methods in it, it returns a empty list:[].
So this is the method with the multiple methods in it:
def show_extracted_data_from_file(self, file_name):
self.extractingText.extract_text_from_image(file_name)
total_fruit = self.filter_verdi_total_number_fruit()
fruit_name = self.filter_verdi_fruit_name()
fruit_total_cost = self.filter_verdi_total_fruit_cost(file_name)
return "\n".join("{} \t {} \t {}".format(a, b, c) for a, b, c in zip(total_fruit, fruit_name, fruit_total_cost))
and this is the method: filter_verdi_total_fruit_cost:
def filter_verdi_total_fruit_cost(self, file_name):
locale.setlocale(locale.LC_ALL, locale='Dutch')
self.extractingText.extract_text_from_image(file_name)
return [
locale.atof(items[-1]) for items in (
token.split() for token in file_name.split('\n')
) if len(items) > 2 and items[1] in self.extractingText.list_fruit
]
this method returns the following data:
[123.2, 2772.0, 46.2, 577.5, 69.3, 3488.16, 137.5, 500.0, 1000.0, 2000.0, 1000.0, 381.25]
You see that I am calling two times file_name.
and so when I calling the method show_extracted_data_from_file in the views.py:
if uploadfile.image.path.endswith('.pdf'):
content = filter_text.show_extracted_data_from_file(uploadfile.image.path)
print(content)
it produces a empty list: []
Question: how can I reduce the parameter file_name so that it will return the correct results?
this are my two other methods that I am calling in the combined method:
def filter_verdi_total_number_fruit(self):
regex = r"(\d*(?:\.\d+)*)\s*\W+(?:" + '|'.join(re.escape(word)
for word in self.extractingText.list_fruit) + ')'
return re.findall(regex, self.extractingText.text_factuur_verdi[0])
def filter_verdi_fruit_name(self):
regex = r"(?:\d*(?:\.\d+)*)\s*\W+(" + '|'.join(re.escape(word)
for word in self.extractingText.list_fruit) + ')'
return re.findall(regex, self.extractingText.text_factuur_verdi[0])
So this is the other class:
class ExtractingTextFromFile:
def extract_text_from_image(self, filename):
self.text_factuur_verdi = []
pdf_file = wi(filename=filename, resolution=300)
all_images = pdf_file.convert('jpeg')
for image in all_images.sequence:
image = wi(image=image)
image = image.make_blob('jpeg')
image = Image.open(io.BytesIO(image))
text = pytesseract.image_to_string(image, lang='eng')
self.text_factuur_verdi.append(text)
return self.text_factuur_verdi
def __init__(self):
# class variables:
self.tex_factuur_verdi = []
self.list_fruit = ['Appels', 'Ananas', 'Peen Waspeen',
'Tomaten Cherry', 'Sinaasappels',
'Watermeloenen', 'Rettich', 'Peren', 'Peen',
'Mandarijnen', 'Meloenen', 'Grapefruit', 'Rettich']
#AndrewRyan has the right idea.
I presume calling extract_text_from_image just adds the attribute list_fruit
Two routes you can go, from what you are commenting you'll probably just go with #1.. but I gave #2 as another option in case you'd ever want to call filter_verdi_total_fruit_cost by itself.
Path 1, Just remove it.
Note: filter_verdi_total_fruit_cost is only called from show_extracted_data_from_file.
def show_extracted_data_from_file(self, file_name):
# extract text
# Note: stores data in `self.extractingText.list_fruit`
self.extractingText.extract_text_from_image(file_name)
total_fruit = self.filter_verdi_total_number_fruit()
fruit_name = self.filter_verdi_fruit_name()
fruit_total_cost = self.filter_verdi_total_fruit_cost()
return "\n".join("{} \t {} \t {}".format(a, b, c) for a, b, c in zip(total_fruit, fruit_name, fruit_total_cost))
def filter_verdi_total_fruit_cost(self):
# Note: `self.extractingText.list_fruit` should be already defined
locale.setlocale(locale.LC_ALL, locale='Dutch')
return [
locale.atof(items[-1]) for items in (
token.split() for token in file_name.split('\n')
) if len(items) > 2 and items[1] in self.extractingText.list_fruit
]
Path 2, Check if it's already extracted- if not, extract; if so, continue
Note: if you wanted to just call filter_verdi_total_fruit_cost
def show_extracted_data_from_file(self, file_name):
# extract text
# Note: stores data in `self.extractingText.list_fruit`
self.extractingText.extract_text_from_image(file_name)
total_fruit = self.filter_verdi_total_number_fruit()
fruit_name = self.filter_verdi_fruit_name()
fruit_total_cost = self.filter_verdi_total_fruit_cost(file_name)
return "\n".join("{} \t {} \t {}".format(a, b, c) for a, b, c in zip(total_fruit, fruit_name, fruit_total_cost))
def filter_verdi_total_fruit_cost(self, file_name):
locale.setlocale(locale.LC_ALL, locale='Dutch')
if not hasattr(self, 'list_fruit'):
# file hasn't been extracted yet.. extract it
# Note: stores data in `self.extractingText.list_fruit`
self.extractingText.extract_text_from_image(file_name)
return [
locale.atof(items[-1]) for items in (
token.split() for token in file_name.split('\n')
) if len(items) > 2 and items[1] in self.extractingText.list_fruit
]
I download the source html code from https://down.ali213.net/pcgame/all/falcom-0-0-0-new-pic-1 as the file htmlstr1.html
Then I use the following python code to handle the htmlstr1.html, in the code, I am trying to delete all the {a} tags and {span} tags with the bs4 method extract. However I notice there are still a few {a} tags and {span} tags in the output file htmlstr_extracted1.html
I dont't know where goes wrong, anybody can help?
# 抽取html的特征
from bs4 import BeautifulSoup
from bs4 import NavigableString, Tag, Comment
import re
import hashlib
def read_file(_path):
with open(_path, "r", encoding="utf-8") as f:
_html_str = f.read()
return _html_str
def write_file(_path, _str):
with open(_path, "w", encoding="utf-8") as f:
f.write(_str)
def md5_str(_str):
md = hashlib.md5(_str.encode())
return md.hexdigest()
class HtmlFeatureExtractor:
def __init__(self, _html_str):
self.del_col = []
_html_str = self.replace_spaces_with_space(_html_str)
_html_str = self.remove_all_br(_html_str)
self.soup = BeautifulSoup(_html_str, 'lxml')
def inline_block_check(self, parent_obj):
if isinstance(parent_obj, Tag):
if parent_obj.name in ['p', 'a', 'span', 'h3', 'b', 'h1', 'strong', 'font', 'h1', 'h4', 'li'] and len(
parent_obj.parent.contents) == 1:
self.collect_del_element(parent_obj)
self.inline_block_check(parent_obj.parent)
def collect_del_element(self, element):
if element not in self.del_col:
self.del_col.append(element)
def remove_all_br(self, _str):
_str = _str.replace("\r\n", "")
_str = _str.replace("\r", "")
_str = _str.replace("\n", "")
return _str
def replace_spaces_with_space(self, _str):
pattern = re.compile(r'\s+', re.I | re.S)
_str = pattern.sub(' ', _str)
return _str
def not_others_child(self, element):
for i in self.del_col:
if element is i:
continue
if isinstance(i, Tag) or isinstance(i, NavigableString):
try:
for ii in i.descendants:
if element is ii:
return False
except Exception as err:
continue
return True
def extract(self):
for x in self.soup.descendants:
if isinstance(x, NavigableString):
self.collect_del_element(x)
if isinstance(x.parent, Tag):
if len(x.parent.contents) == 1 and x.parent.name not in ['[document]', 'html', 'body']: # 对inline部分再搞一层，并且子集唯一
self.collect_del_element(x.parent) # 第一层，直接删
self.inline_block_check(x.parent.parent) # 再往后，递归
else:
if x.name in ['script', 'meta', 'link', 'style', 'img', 'a', 'input', 'iframe', 'form', 'p', 'li', 'span']: # 还要包含a标签， ！！！写一个去重判断
self.collect_del_element(x) # 后删
if x.name in ['img', 'a', 'iframe']:
self.inline_block_check(x.parent)
self.del_col = [i for i in self.del_col if self.not_others_child(i)]
for y in self.del_col:
if isinstance(y, Tag) or isinstance(y, NavigableString):
y.extract()
# z = y.extract()
# print("-" * 30)
# print(z)
_feature_str = str(self.soup)
_feature_str = self.replace_spaces_with_space(_feature_str)
_feature_str = self.remove_all_br(_feature_str)
write_file("./htmlstr_extracted%d.html" % gg, _feature_str) # todo 调试用
return md5_str(_feature_str)
if __name__ == '__main__':
g_num = [1]
for gg in g_num:
html_str = read_file("./htmlstr%d.html" % gg)
feature_extractor = HtmlFeatureExtractor(html_str)
feature_str = feature_extractor.extract()
print("成功: %s" % feature_str)
my purpose is to extract the feature of a html source, I wish to get the same hash value if the webpage use the same template. the following two pages use the same template, so they should have the same hash value calculate by the above code.
https://down.ali213.net/pcgame/all/falcom-0-0-0-new-pic-1
https://down.ali213.net/pcgame/all/rockstar-0-0-0-new-pic-1
The title says it all - I am looking for a way (or ways) to load, enable, add (and likewise: disable / remove) extensions after the webdriver has been created.
Context: Python, selenium (per tags). Pyautogui for certain parts would be unavoidable, but would need to be robust. I have a draft working soln but its rough and a little GPU dependent (point: can/will not accept "this cannot be done with Chrome" as I know it can be done, but would like to see some soln/code to this effect).
I am perfectly aware of the various methods to load an extension prior to the driver being instantiated / created. i.e.
o.add_extension(crx file location)
o.add_argument(unpacked folder loc)
o.add_argument('--user-data-dir=' + profileX)
where o = Options, profileX has extensions pre-installed and so forth. So will be clear - I am completely uninterested in proposals to this effect.
Any ideas/proposals in regards to the question in the title/above?
Ideally (but not absolutely critical) would be a programmatic solution that is not entirely GUI based (robotic/PyAutogui dependent).
You can assume the extension is available on the chrome store for this exercise.
Use-case: The extension I'm testing involves registering points and to protect fair / intended usage terms, its designed to cease functioning whenever it recognises automated manipulation (which is perfectly possible with 'headed' chrome, e.g. you just assign it a hotkey in chrome settings, and use Pyautogui to activate, or locate it's html page and interact directly etc.). GIVEN this, and the very many, many tests I have, it seems a waste of resource me having to reload the tests, or set up profiles, or stop and start the tests every so often so that I can include 'fresh' copies of the crx files or deploy one of the other methods I mention above. To date / my best knowledge, there is no solution for this. Happy to be pleasantly surprised if someone can point me to a 'duplicate' Q in this regard.
NOTES
Problem addressed with three possible approaches (thus soln. appears lengthy, but individual methods are not excessive/particular lengthy; what's more, code can be easily adapted as req.)
Further, this starts in the context of a single-browser environment / setup;
Ensues with variable and method definitions that extend the premise / thinking to encompass implementation within a multi-threaded / multi-browser environment
PRELIMINARY
import glob2,itertools,os, pyautogui, shutil, time, pyperclip, subprocess, keyboard as kb
from datetime import datetime
from zipfile import ZipFile
now = datetime.now()
dt_string: str = now.strftime("%d-%b-%Y, %H:%M:%S")
print(dt_string)
from selenium import webdriver
from selenium.webdriver import DesiredCapabilities
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from concurrent.futures import ThreadPoolExecutor
from os import path
#references: date-time format: https://www.programiz.com/python-programming/datetime/current-datetime
datetime reference
METHODS
Method 1: add after chrome launches via Chrome-store
def ext_update_method1(ps, section=''):
global w, p2s
text_temp, p2s = [''] * len(ps), []
def button_click(p, parms = ['',True]):
#parms[0] = ID, parms[1] = click?
ID,click = parms[0], parms[1]
global text_temp
text_temp[p] = ''
ID = ['omghfjlpggmjjaagoclmmobgdodcjboh'] if ID == '' else ID #default i= Browsec if no ID entered
d[p].get(f'https://chrome.google.com/webstore/detail/{ID[0]}')
w[p] = WebDriverWait(d[p], 10)
w[p].until(lambda x: x.execute_script("return document.readyState") == "complete")
button = w[p].until(lambda x: x.find_element_by_class_name('g-c-R'))
text_temp[p] = button.text
if click == True: button.click()
def add_remove():
global p2s
for i, y in enumerate(ys): activ(i, ys), (ss, 0.5), keys([(pr, 'tab'), (pr, 'space')])
after2, ys2 = gwwt(''), []
for x in after2:
if (x not in before) and (x not in ys_remove): ys2 = ys2 + [x, ]
# ys2 = ys2 + [x,] if (all(x not in before) and (x not in ys_remove)) else ys2
for i, y in enumerate(ys2): activ(i, ys), (ss, 0.5), keys([(pr, 'tab'), (pr, 'space')])
ss(1)
after2, ys2 = gwwt(''), []
for x in after2:
if (x not in before) and (x not in ys_remove) and (x not in a[0] for a in wd): ys2 = ys2 + [x, ]
for i, y in enumerate(ys2):
if all(y != x[0] for x in wd): y.close()
if len(ys_remove) > 0:
for i, y in enumerate(ys_remove):
activ(i, ys_remove), pr('space'), ss(0.5)
p2s = p2s + [i, ]
if section == 1:
return 'complete 2nd iteration' #prevent endless loop (see next comment)
else:
print('removed add-in, now adding back again....')
cc(2)
thread_all(ps, close_extra)
ext_update_method1(p2s, 1) #loop back on self to add
else:
return 'complete 1st iteration'
def close_extra(p):
global w
w_old = w[p]
w[p] = WebDriverWait(d[p], 10)
tabs_close_extra(p)
tabs_close_extra(p)
activ(p), pr('a'), activ(p), pr('enter')
w[p].until(lambda x: x.execute_script("return document.readyState") == "complete")
w[p] = w_old
thread_all(ps, tabs_close_extra)
before = gwwt('')
thread_all(ps, button_click)
cc(2) #cc(5)
after = gwwt('')
ys, ys_remove, p2s = [], [], []
ys_remove = gwwt('remove')
print(f'ys_remove = {ys_remove}')
for x in after:
if (x not in ys_remove) and (x not in before): ys = ys + [x, ]
print(add_remove())
cc(2)
thread_all(ps, close_extra)
thread_all(ps,button_click,['',False])
qs = []
for x in text_temp: qs = qs + [1,] if 'add' in x.lower() else qs
print(f'qs: {qs}')
if section == 2: return #prevent never-ending loop (see next comment)
if len(qs) > 0: ext_update_method1(qs,2) #loop back on self in case 'add extension' still present in web-store
Notes:
Can download multiple versions of same extension on same browser
Caution: 3rd parameter in n_i_c is set to 1 by default: this will
clear your downloads folder. Set to 0 if don't want this to happen
Arguments:
args: n_i_c[0] = number of times you wish to download a given
extension onto the same browser
n_i_c[1] = item of global list: ext_details, see vars()n_i_c2 =
1 to clear folder
n_i_c[3]= 0 => only download unpacked folder (not loading it too)
Requires methods ('Ancillary Code', Vars) below - code can be modified
to exclude these.
Method 2: Download & load .crx file after Chrome launch
def ext_update_method2(ps, n_i_c=[2, 0, 1,1], section = ''): # same order as it would
appear for single set of extensions (iro one tab)...
# parms = [2,0,0,0] =>
# 0. repetitions - e.g. 2 downloads : uBlock
# 1. 0 [index value - e.g. uBlock = 4, below] downloaded and unzipped/installed on profile p
# 2: clear folder
# 3: is_load = 0 => only download unpacked folder (not loading it too)
global start, w, files_old, ev, d, parent, txt
global tabs_before
global files, files_new, index_latest_files, index_redo, index_hks
hks2, index_ext, clear_folder, is_load = n_i_c[0], n_i_c[1], n_i_c[2], n_i_c[3] # ID, alias = cred[0], cred[1], cred[2]
hks2 = hks if hks == '' else hks2
ID, alias = ext_details[index_ext][1], ext_details[index_ext][0]
url_robwu = '''https://robwu.nl/crxviewer/?crx=https://chrome.google.com/webstore/detail/''' + str(ID)
if clear_folder == 1:
folder_clear(del_dir=os.path.join(os.path.expanduser("~"), 'Downloads', '*'))
def thread_all_0(p): #set up stuff
global d, w, index_hks2
if 'error' in str(tabs_close_extra(p)):
tab_close(p)
d[p].implicitly_wait(5)
w[p] = WebDriverWait(d[p], 5)
index_hks2[p] = ''
d[p].get('chrome://extensions')
code_0 = '''var tags = document.querySelector('body').querySelector('extensions-manager')['shadowRoot'].querySelector('cr-view-manager').querySelector('extensions-item-list')["shadowRoot"].querySelector('#container').querySelectorAll('extensions-item'); return tags.length;'''
index_hks[p]=ev[p](code_0)
print(index_hks[p])
#
thread_all(ps, thread_all_0)
#
def thread_all_1(p): #open website to download crx (multi-thread to expedite)
d[p].get(url_robwu)
#
if len(ps) >= 2: #prevent overloading website - only do 3 at a time if more than 2 browsers being used
groups = list(data_mygrouper(2, ps))
for x in groups: thread_all(x, thread_all_1,workers = 3, chunk = 3)
else:
thread_all(ps, thread_all_1)
files_old = glob2.glob(os.path.join(os.path.expanduser("~"), 'Downloads', '*.zip'))
def thread_all_2(p):
try:
w[p].until(lambda x: x.find_element_by_id("download-link")).click()
print('lambda wait passed !!')
except:
print('lambda wait failed :(')
return
start_temp = time.time()
while (time.time() - start_temp <= 90) and (len(glob2.glob(os.path.join(os.path.expanduser("~"), 'Downloads', '*.zip'))) - len(files_old) < hks2 * (len(ps))) or (kb.is_pressed('escape') == True):
if kb.is_pressed('escape') == True: break
thread_all(ps, thread_all_2), ss(1)
files = glob2.glob(os.path.join(os.path.expanduser("~"), 'Downloads', '*.zip'))
files_new = []
for keep in files:
if str(keep) not in files_old: files_new = files_new + [str(keep), ]
# thread_all(files_new, z_extension_unzip, alias)
path_dir_core = os.path.join(os.path.expanduser("~"), 'Downloads')
index_latest_files = [''] * len(files_new)
#
def thread_all_3(j): # unzip
global index_latest_files
index_latest_files[j] = os.path.join(path_dir_core, alias + str(j))
with ZipFile(files_new[j], 'r') as zipObj:
zipObj.extractall(index_latest_files[j])
os.remove(files_new[j])
#
thread_all(list(range(len(files_new))), thread_all_3)
if is_load != 1: return
code1 = '''if(document.querySelector('body').querySelector('extensions-manager')["shadowRoot"].querySelector('extensions-toolbar').attributes.length !=2){document.querySelector("body").querySelector("extensions-manager")["shadowRoot"].querySelector("extensions-toolbar")["shadowRoot"].querySelector("cr-toolbar").querySelector(".more-actions").querySelector("#devMode")["shadowRoot"].querySelector("#knob").click() } else {'dev mode already activated!'}'''
code2 = '''document.querySelector('body').querySelector('extensions-manager')["shadowRoot"].querySelector('extensions-toolbar')["shadowRoot"].querySelector("#loadUnpacked").click()'''
#
def thread_all_4(p):
d[p].get('chrome://extensions/'), ss(1)
ev[p](code1)
for y in range(hks2):
thread_all(ps, thread_all_4)
windows_before = gwwt('Select the extension directory')
for p in ps:
try:
if path.exists(os.path.join(path_dir_core, index_latest_files[p + len(ps) * (y)])):
try:
ev[p](code2)
except:
continue
start_temp = time.time()
while (time.time() - start_temp <= 30) and (
len(gwwt('Select the extension directory')) - len(windows_before)) < 1 and (
'select the extension directory' not in gawt().lower()): pass # len(ps) - len(errors_chrome()): pass
tabs_before = len(d[p].window_handles)
windows_A = gwwt('')
try:
keys([(tw, index_latest_files[p + len(ps) * (y)]), (ss, 0.5), (pr, 'enter'), (pr, 'tab'),(pr, 'space')], 0.75)
if len(windows_A) < len(gwwt('')):
d[p].refresh()
print('would break1')
else:
window_target = gwwt('Select the extension directory')[0]
try:
window_target.activate()
except:
window_target.minimize()
window_target.restore()
finally:
pyperclip.copy(index_latest_files[p + len(ps) * (y)]), ss(0.5)
keys([(hk, 'ctrl', 'l'), (kd, 'shift'), (pr, 'tab', 4), (ku, 'shift'), (hk, 'ctrl', 'v'), (pr, 'enter'), (pr, 'tab'), (pr, 'space')])
except:
d[p].refresh()
pr('escape', 2)
print('would break2')
tabs_close_extra(p)
else: continue
except: continue
def thread_all_5(p):
global qs
d[p].get('chrome://extensions/shortcuts')
code_0 = '''return document.querySelector('body').firstElementChild["shadowRoot"].querySelector("cr-view-manager").querySelector('extensions-keyboard-shortcuts')['shadowRoot'].querySelector('#container').querySelectorAll('.shortcut-card').length'''
index_hks[p] = int(ev[p](code_0)) - index_hks[p]
qs = qs + [p,] if index_hks[p] < hks2 else qs
print(f'index_hks[{p}] = {index_hks[p]}')
index4[p] = 0
qs = []
if clear_folder == 1:
thread_all(ps, thread_all_5)
if section == 1: return
if len(qs) > 0: ext_update_method2(qs, n_i_c = [hks2, index_ext, 0, 1], section = 1)
qs = []
for p in range(len(ps)): qs = qs + [p,]
Method 3: remove programmatically
def ext_remove(ps,ext):
for p in range(len(ps)):
try:
activ(p)
d[p].get('chrome://extensions/')
#pr('f12'),ss(1)
hk('ctrl', 'shift', 'j'), ss(1)
code = '''var tags = document.querySelector('body').querySelector('extensions-manager')['shadowRoot'].querySelector('cr-view-manager').querySelector('extensions-item-list')["shadowRoot"].querySelector('#container').querySelectorAll('extensions-item'); for (i = 0;i <= tags.length; i ++) {try {if(tags[i]['shadowRoot'].querySelector('#a11yAssociation').innerText.toLowerCase().indexOf("''' + str(ext) +'''") > 0) {tags[i]['shadowRoot'].querySelector('#remove-button').click();break} } catch {}}'''
pyperclip.copy(code), ss(0.5)
# pr('tab'), ss(0.5)
temp_rep = 1 if type(ext) == str else len(ext)
for _ in range(temp_rep):
hk('ctrl', 'v'), ss(0.5)
pr('enter'), ss(0.5)
pr('space'), ss(0.5)
tabs_close_extra(p)
pr('f12'), ss(0.5) #hk('ctrl', 'shift', 'j'), ss(0.5)
except: pass
ANCILLARY CODE
Activate window
Shorthand: simulate keyboard strokes
Methods: sub-divide lists, close tabs, thread-pool (multi-threading), and tile windows*
*(java scripts incl. - to be saved in same location as this script
Activate window
pyautogui.FAILSAFE = False
def activ(p, index_wd=''):
global index_activ
try:
if index_wd == '':
index_wd = wd[p][0]
else:
index_wd = index_wd[p]
except:
index_wd = wd[p]
print(f'active {w}')
index_activ[p] = False
try:
index_wd.activate()
index_activ[p] = True
return index_activ[p]
except:
try:
index_wd.minimize()
index_wd.restore()
index_activ[p] = True
return index_activ[p]
except:
index_activ[p] = False
index_activ[p] = False
return index_activ[p]
Quick keys
def keys(actions, delay=0.5):
# global BL3
print(f'def keys()') # {actions}') #, profile {p}')
outcome = []
for action in actions:
print(f'action {action}')
if len(action) == 4:
outcome = outcome + [action[0](action[1], action[2], action[3]), ]
elif len(action) == 3:
outcome = outcome + [action[0](action[1], action[2]), ]
elif (len(action) == 2) and (action[1] != ''):
outcome = outcome + [action[0](action[1]), ]
else:
outcome = outcome + [action[0](), ]
ss(delay) # await ass(delay)
return outcome
Sub-divide lists
def data_mygrouper(n,
iterable):
args = [iter(iterable)] * n
return ([e for e in t if e != None] for t in itertools.zip_longest(*args))
stack overflow reference (data_mygrouper)
ThreadPool
def thread_all(ps, fn, parm='', actions=[], workers=6, chunk=1):
# https://stackoverflow.com/questions/42056738/how-to-pass-a-function-with-more-than-one-argument-to-python-concurrent-futures/42056975
print(f'thread_all({ps}, {fn}, {parm}, {actions}')
if parm == '':
with ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
return executor.map(fn, ps, timeout=90, chunksize=max(1, chunk))
else:
with ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
return executor.map(fn, ps, itertools.repeat(parm, L), timeout=90, chunksize=max(1, chunk))
thread-pool_exector ref: here
close tabs
def tab_close(p):
print(f'def tab_close {p}')
# d[p].switch_to.window(parent[p])
for h in d[p].window_handles:
d[p].switch_to.window(h)
parent[p] = h
[ev[p]] = [d[p].execute_script]
break
# parent[p] = d[p].current_window_handle
for h in d[p].window_handles:
if h != parent[p]:
d[p].switch_to.window(h)
if len(d[p].window_handles) > 1: d[p].close()
else: break
d[p].switch_to.window(parent[p])
def tabs_close_extra(p):
# alternative method (much slower): https://stackoverflow.com/questions/12729265/switch-tabs-using-selenium-webdriver-with-java
try:
for h in d[p].window_handles:
d[p].switch_to.window(h)
parent[p] = h
break
# parent[p] = d[p].current_window_handle
tabs_original = len(d[p].window_handles)
if tabs_original > 1:
for h in d[p].window_handles:
if h != parent[p]:
d[p].switch_to.window(h)
activ(p), hk('ctrl', 'w')
if len(d[p].window_handles) == tabs_original:
d[p].close()
# d[p].switch_to.window(parent[p])
d[p].switch_to.window(parent[p])
except:
print(f'error in tabs_close_extra {p}')
Variables
def vars():
print('def vars()')
global index, index2, index3, index4, index5, index6
global index_activ, index_activ2, hks, pycharm_win
global index_latest_files, index_hks, index_hks2
global parent, d, w, w2
global path_chrome, path_exec, path_core, path_temp, profile, done, caps, pycharm_win
global ps, p2s, qs
global L, tile_type, compact, outcome
global kd, ku, hk, pr, tw, wr, cl
global gats, gaws, gwwt, gawt, gaw
global cc, ss, start, start2, start_overall
global wd, wd2, ev, errors, errors2, errors3
global ext_0, ext_1, ext_2, ext_3,ext_details
caps = DesiredCapabilities.CHROME.copy()
[cc, ss, kd, ku, hk, pr, tw, wr, cl] = [pyautogui.countdown, time.sleep,
pyautogui.keyDown, pyautogui.keyUp, pyautogui.hotkey, pyautogui.press,,pyautogui.typewrite, pyautogui.write, pyautogui.click]
[gats, gaws, gwwt] = [pyautogui.getAllTitles, pyautogui.getAllWindows, pyautogui.getWindowsWithTitle]
[gawt, gaw] = [pyautogui.getActiveWindowTitle, pyautogui.getActiveWindow]
L = 4
qs, p2s, ps, = list(range(L)), list(range(L)), list(range(L))
text_temp, start = [''] * L, [0] * L
index_hks, index_hks2 = [hks] * L, [0] * L
index_latest_files = ['']
parent, w, ev, d, wd = [''] * L, [''] * L, [''] * L, [''] * (L + 1), [['']]*L
start_overall, outcome = '', ''
try: pycharm_win = [gaw()]
except: pass
path_chrome = "C:\Program Files\Google\Chrome\Application\chrome.exe"
path_core = os.path.join(os.path.expanduser("~"), 'PyCharmProjects', 'Parallel')
path_exec = os.path.join(path_core, "chromedriver.exe")
ext_details = [ ['ublock', 'cjpalhdlnbpafiamejdnhcphjbkeiagm'], ['keyboard_shortcuts', 'dkoadhojigekhckndaehenfbhcgfeepl'], ['browsec', 'omghfjlpggmjjaagoclmmobgdodcjboh'], ['itrace', 'njkmjblmcfiobddjgebnoeldkjcplfjb']]
ext_0= os.path.join(path_core, "Browsec")
ext_1 = os.path.join(path_core, "Keyboard")
ext_2 = os.path.join(path_core, "itrace")
ext_3 = os.path.join(path_core, "ublock")
MAIN CODE
if __name__ == '__main__':
ps =''
vars()
if len(ps) > 3:
ps_new = list(data_mygrouper(int(round(len(ps) / 2, ndigits=0)), ps))
for x in range(2): thread_all(ps_new[x], new_profile, workers=len(ps_new[x]) + 1, chunk=len(ps_new[x]) + 1), cc(5)
# new_profile
else: thread_all(ps, new_profile, workers=L + 1, chunk=L+1)
#thread_all(ps,new_profile)
tile_windows('v')
ext_update_method1(ps)
ext_remove(ps,'browsec')
ext_update_method2(ps)
I tried to modify the SIR model from the eon package and did some changes to it. It has a new vaccination parameter attached to it with new parameters beta and omega and Vl and my code is-
def test_transmission(u, v, p):
return random.random()<p
def discrete_SIR(G,
initial_infecteds,beta,
w,Vl,return_full_data=True):
if G.has_node(initial_infecteds):
initial_infecteds=[initial_infecteds]
if return_full_data:
node_history = defaultdict(lambda : ([tmin], ['S']))
transmissions = []
for node in initial_infecteds:
node_history[node] = ([tmin], ['I'])
transmissions.append((tmin-1, None, node))
node_history = defaultdict(lambda : ([tmin], ['S']))
# transmissions = []
for node in initial_infecteds:
node_history[node] = ([tmin], ['I'])
#transmissions.append((tmin-1, None, node))
N=G.order()
t = [tmin]
S = [N-len(initial_infecteds)]
I = [len(initial_infecteds)]
R = [0]
V = [0]
susceptible = defaultdict(lambda: True)
#above line is equivalent to u.susceptible=True for all nodes.
for u in initial_infecteds:
susceptible[u] = False
infecteds = set(initial_infecteds)
while infecteds and t[-1]<tmax :
new_infecteds = set()
vaccinated= set()
infector = {} #used for returning full data. a waste of time otherwise
for u in infecteds:
# print('u-->' +str(u))
for v in G.neighbors(u):
# print('v --> '+ str(v))
##vaccination
if len(vaccinated)+V[-1]< (Vl*N) : #check if vaccination over or not
#print(len(vaccinated),Vl*N)
#print("HI")
if susceptible[v] and test_transmission(u, v, w):
vaccinated.add(v)
susceptible[v] = False
# print('transmitting vaccination')
elif susceptible[v] and test_transmission(u,v,beta):
new_infecteds.add(v)
susceptible[v]=False
infector[v] = [u]
# print('transmitting infection')
else:
# print("BYE")
if susceptible[v] and test_transmission(u, v,beta):
new_infecteds.add(v)
susceptible[v] = False
infector[v] = [u]
#infector[v] = [u]
if return_full_data:
for v in infector.keys():
transmissions.append((t[-1], random.choice(infector[v]), v))
next_time = t[-1]+1
if next_time <= tmax:
for u in infecteds:
node_history[u][0].append(next_time)
node_history[u][1].append('R')
for v in new_infecteds:
node_history[v][0].append(next_time)
node_history[v][1].append('I')
infecteds = new_infecteds
R.append(R[-1]+I[-1])
V.append(len(vaccinated)+V[-1])
I.append(len(infecteds))
S.append(N-V[-1]-I[-1]-R[-1])
#S.append(S[-1]-V[-1]-I[-1])
t.append(t[-1]+1)
print(str(R[-1])+','+str(V[-1])+','+str(I[-1])+','+str(S[-1]))
if not return_full_data:
return scipy.array(t), scipy.array(S), scipy.array(I), \
scipy.array(R)
else:
return EoN.Simulation_Investigation(G, node_history, transmissions)
Now I want to run the visualizations on it like in the packagae EON-
m=5
G=nx.grid_2d_graph(m,m,periodic=True)
initial_infections = [(u,v) for (u,v) in G if u==int(m/2) and v==int(m/2)]
sim = EoN.basic_discrete_SIR(G,0.5,initial_infecteds = initial_infections,
return_full_data=True, tmax = 25)
pos = {node:node for node in G}
sim.set_pos(pos)
sim.display(0, node_size = 40) #display time 6
plt.show()
plt.savefig('SIR_2dgrid.png')
What changes do I need to do in my code so that the display function works or do I need to make changes in the display function also?
Here's the output I now get:
You'll have to install EoN version 1.0.8rc3 or later, which is available on the github page (see installation instructions). At present pip will not work to install it. I want to make sure I haven't broken anything before I make it the default installed by pip.
Here's the code based on yours. You should look through the changes I've made. It's also worth looking at the examples I've put in the documentation (including an SIRV model where the vaccination rule is different than what you've got).
from collections import defaultdict
import EoN
import networkx as nx
import random
import matplotlib.pyplot as plt
def test_transmission(u, v, p):
return random.random()<p
def discrete_SIRV(G, initial_infecteds,beta,
w,Vl,tmin=0,tmax=float('Inf'), return_full_data=True):
if G.has_node(initial_infecteds):
initial_infecteds=[initial_infecteds]
if return_full_data:
node_history = defaultdict(lambda : ([tmin], ['S']))
transmissions = []
for node in initial_infecteds:
node_history[node] = ([tmin], ['I'])
transmissions.append((tmin-1, None, node))
'''
node_history = defaultdict(lambda : ([tmin], ['S']))
# transmissions = []
for node in initial_infecteds:
node_history[node] = ([tmin], ['I'])
#transmissions.append((tmin-1, None, node))
'''
N=G.order()
t = [tmin]
S = [N-len(initial_infecteds)]
I = [len(initial_infecteds)]
R = [0]
V = [0]
susceptible = defaultdict(lambda: True)
#above line is equivalent to u.susceptible=True for all nodes.
for u in initial_infecteds:
susceptible[u] = False
infecteds = set(initial_infecteds)
while infecteds and t[-1]<tmax :
new_infecteds = set()
vaccinated= set()
infector = {} #used for returning full data. a waste of time otherwise
for u in infecteds:
# print('u-->' +str(u))
for v in G.neighbors(u):
# print('v --> '+ str(v))
##vaccination
if len(vaccinated)+V[-1]< (Vl*N) : #check if vaccination over or not
#print(len(vaccinated),Vl*N)
#print("HI")
if susceptible[v] and test_transmission(u, v, w):
vaccinated.add(v)
susceptible[v] = False
'''It's probably better to define a `new_vaccinated`
set and then do the `return_full_data` stuff later
where all the others are done.'''
if return_full_data:
node_history[v][0].append(t[-1]+1)
node_history[v][1].append('V')
# print('transmitting vaccination')
elif susceptible[v] and test_transmission(u,v,beta):
new_infecteds.add(v)
susceptible[v]=False
infector[v] = [u]
# print('transmitting infection')
else:
# print("BYE")
if susceptible[v] and test_transmission(u, v,beta):
new_infecteds.add(v)
susceptible[v] = False
infector[v] = [u]
#infector[v] = [u]
if return_full_data:
for v in infector.keys():
'''This random choice is no longer needed as you've taken out
the possibility of multiple nodes transmitting to `v` in a given
time step. Now only the first one encountered does it.'''
transmissions.append((t[-1], random.choice(infector[v]), v))
next_time = t[-1]+1
if next_time <= tmax:
for u in infecteds:
node_history[u][0].append(next_time)
node_history[u][1].append('R')
for v in new_infecteds:
node_history[v][0].append(next_time)
node_history[v][1].append('I')
infecteds = new_infecteds
R.append(R[-1]+I[-1])
V.append(len(vaccinated)+V[-1])
I.append(len(infecteds))
S.append(N-V[-1]-I[-1]-R[-1])
#S.append(S[-1]-V[-1]-I[-1])
t.append(t[-1]+1)
print(str(R[-1])+','+str(V[-1])+','+str(I[-1])+','+str(S[-1]))
if not return_full_data:
return scipy.array(t), scipy.array(S), scipy.array(I), \
scipy.array(R)
else:
return EoN.Simulation_Investigation(G, node_history, transmissions, possible_statuses=['S', 'I', 'R', 'V'])
print(EoN.__version__)
print("line above needs to be 1.0.8rc3 or greater or it will not work\n\n")
m=20
G=nx.grid_2d_graph(m,m,periodic=True)
initial_infections = [(u,v) for (u,v) in G if u==int(m/2) and v==int(m/2)]
beta=0.8
Vl=0.3
w=0.1
sim = discrete_SIRV(G, initial_infections, beta, w, Vl, return_full_data=True)
pos = {node:node for node in G}
sim.set_pos(pos)
sim.sim_update_colordict({'S': '#009a80','I':'#ff2000', 'R':'gray','V': '#5AB3E6'})
sim.display(6, node_size = 40) #display time 6
plt.savefig('SIRV_2dgrid.png')