We Keep Coding

concurrent.futures.ProcessPoolExecutor(): json file is not created

I'm new to this package. When studying the codes from https://github.com/diningphil/graph-mixture-density-networks (with some minor modification). In the notebook file SIR Simulation with DGL_ERDOS-RENYI.ipynb, during the simulation process, I encountered a weird thing:
If I set debug = True, which means I'm not using the pool = concurrent.futures.ProcessPoolExecutor(max_workers=processes) but just run it one by one, both the .json files and the .bin files will be created according to the json_filepath variable.
However, when I deleted the output and run it by setting debug = False so all the codes will run simultaneously if my understanding is correct, but the json_file will not be created and the program seems to terminate at the step graph.to(torch.device(device)) as all my print command is not executed afterward. I only have the .bin files created.
Could anyone help me by telling me the possible reason or waht I should do about it? Thanks a lot!
'''
run simulation and store
1) state of all nodes at each time step
into a single pandas dataframe for all beta, gamma and repetitions
2) R_0
3) number of total people infected (total - susceptible at the end of the iteration)
'''
seed = 38
torch.manual_seed(seed)
device = 'cuda'
beta_range = [0, 1]
gamma_range = [0.1, 1]
iterations = 5
no_graph_samples = 20
no_realizations = 100
family_name = 'erdos_renyi'
folder = Path(f'{family_name}')
if not os.path.exists(folder):
os.makedirs(folder)
def simulate(p, graph_size, graph_sample, graphs_folder):
json_filepath = str(Path(graphs_folder, f'data_{graph_sample}.json'))
graph_filename = graphs_folder / Path(f'sample{graph_sample}.bin')
json_data = {'family': family_name,
'p': p,
'graph_size': graph_size,
'no_graph_samples': no_graph_samples,
'graph_samples': []
}
sample = {'graph_filename': str(graph_filename),
'simulations': []}
if not os.path.exists(graph_filename):
graph = create_erdos_renyi_graph(graph_size, p)
save_graphs(str(graph_filename), graph)
else:
graph = load_graphs(str(graph_filename))[0][0]
#print('test')
graph.to(torch.device(device))
## every code above this line will run, at least print() will work
if not os.path.exists(json_filepath):
print('test: json_does not exit')
for realizations in range(no_realizations):
beta = float(torch.FloatTensor(1).uniform_(beta_range[0], beta_range[1]))
gamma = float(torch.FloatTensor(1).uniform_(gamma_range[0], gamma_range[1]))
R0 = beta/gamma
graph.register_message_func(lambda x: SIR_message_func(beta, x))
graph.register_reduce_func(lambda x: SIR_reduce_func(gamma, x))
for initial_probability_of_infection in [0.01, 0.05, 0.1]:
simulation = {'beta': beta, 'gamma': gamma, 'R0': R0, 'init_infection_prob': initial_probability_of_infection}
S, I, R, first_infected = simulate_SIR(graph, initial_probability_of_infection, iterations)
simulation['S'] = S
simulation['I'] = I
simulation['R'] = R
simulation['first_infected'] = first_infected
simulation['total_infected'] = graph_size - S[-1]
sample['simulations'].append(deepcopy(simulation))
#print("Realization ", realizations, "produced ", graph_size - S[-1], "infected")
json_data['graph_samples'].append(sample)
with open(json_filepath, 'w') as f:
line = json.dumps(json_data)
f.write(line + '\n')
#json.dump(json_data, f)
print('dumped')
else:
print('test: there is json')
print(sample)
# with open(json_filepath, 'r') as f:
# json.load(f)
# print('loaded but why')
debug = False
processes = 100
import concurrent.futures
pool = concurrent.futures.ProcessPoolExecutor(max_workers=processes)
#for graph_size in [10, 50, 100, 200, 500, 1000]:
for graph_size in [10]:
for p in [0.01, 0.05]:
#for p in [0.01, 0.05, 0.1, 0.2, 0.3, 0.5]:
graphs_folder = folder / Path(f'graphs_size{graph_size}_p{float(p)}')
#store each graph in a different folder (create path based on graph size, prob of edge and graph sample)
if not os.path.exists(graphs_folder):
os.makedirs(graphs_folder)
for graph_sample in range(no_graph_samples):
if not debug:
pool.submit(simulate, p, graph_size, graph_sample, graphs_folder)
else: # DEBUG
simulate(p, graph_size, graph_sample, graphs_folder)
pool.shutdown() # wait the batch of configs to terminate

How to fix neat recurrent.py file, neat python library openAI gym

Trying to get the python neat algorithm to work with openAI gym retro.
I am using python3 with the youtube:https://www.youtube.com/watch?v=8dY3nQRcsac&list=PLTWFMbPFsvz3CeozHfeuJIXWAJMkPtAdS&index=8&t=410s
trying to get neat to work with sonic in the openAI env. it seems there is a problem with the recrrent.py file.
Find the code here: https://gitlab.com/lucasrthompson/Sonic-Bot-In-OpenAI-and-NEAT/blob/master/tut2.py
This Is the Error message
File "tut3.py", line 53, in <module>
winner = p.run(eval_genomes)
File "/home/gym/OPAI/lib/python3.6/site-packages/neat/population.py", line 89, in run
fitness_function(list(iteritems(self.population)), self.config)
File "tut3.py", line 41, in eval_genomes
imgarray.append(y)
AttributeError: 'numpy.ndarray' object has no attribute 'append'
Line 89 in the population.py file
self.reporters.start_generation(self.generation)
# Evaluate all genomes using the user-provided function.
fitness_function(list(iteritems(self.population)), self.config)
The tut3 code that I got from #lucas
Just plan neat net work.
import retro
import numpy as np
import pickle
import neat
import cv2
env = retro.make('SonicTheHedgehog-Genesis', 'GreenHillZone.Act1')
imgarray = []
def eval_genomes(genomes, config):
for genome_id, genome in genomes:
ob = env.reset()
ac = env.action_space.sample()
inx, iny, inc = env.observation_space.shape
inx = int(inx/8)
iny = int(iny/8)
net = neat.nn.RecurrentNetwork.create(genome, config)
current_max_fitness = 0
fitness_current = 0
frame = 0
counter = 0
xpos = 0
xpos_max = 0
done = False
while not done:
env.render()
frame +=1
ob = cv2.resize(ob, (inx,iny))
ob = cv2.cvtColor(ob, cv2.COLOR_BGR2GRAY)
ob = np.reshape(ob, (inx,iny))
imgarray = np.ndarray.flatten(ob)
for x in ob:
for y in x:
imgarray.append(y)
nnOutput = net.activate(imgarray)
print(nnOutput)
ob, rew,done, info = env.step(nnOutput)
imgarray.clear()
config = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
neat.DefaultSpeciesSet, neat.DefaultStagnation,
'config-feedforward')
p = neat.Population(config)
winner = p.run(eval_genomes)
Would be great if you guys could help.
I want to fully understand as it is a school project.
Thanks for your help
:))

Your while loop has some errors in it. Make your eval_genomes function look like this below :
def eval_genomes(genomes, config):
for genome_id, genome in genomes:
ob = env.reset()
ac = env.action_space.sample()
inx, iny, inc = env.observation_space.shape
inx = int(inx/8)
iny = int(iny/8)
net = neat.nn.RecurrentNetwork.create(genome, config)
current_max_fitness = 0
fitness_current = 0
frame = 0
counter = 0
xpos = 0
xpos_max = 0
done = False
while not done:
env.render()
frame +=1
ob = cv2.resize(ob, (inx, iny))
ob = cv2.cvtColor(ob, cv2.COLOR_BGR2GRAY)
ob = np.reshape(ob, (inx,iny))
imgarray = np.ndarray.flatten(ob)
nnOutput = net.activate(imgarray)
print(nnOutput)
ob, rew,done, info = env.step(nnOutput)
The ndarray.flatten does the same thing as the for x and for y loop so you only need one of the two solutions and flatten is easier to read. Additionally python is a language where indentation really matters. Always make sure your tabs/spaces are lined up correctly!
Hope that works. If it doesn't, just go on and use this:
https://gitlab.com/lucasrthompson/Sonic-Bot-In-OpenAI-and-NEAT/blob/master/tut2.py
or this:
https://gitlab.com/lucasrthompson/Sonic-Bot-In-OpenAI-and-NEAT/blob/master/neat-paralle-sonic.py
Good luck!

How to fix config Error in python neat openai retro

following the guide at: https://www.youtube.com/watch?v=8dY3nQRcsac&list=PLTWFMbPFsvz3CeozHfeuJIXWAJMkPtAdS&index=7
when I run the python program the is an error in the python neat config file
it looks like it has something to do with the genome variable
my python neat config file now
#--- parameters for the XOR-2 experiment ---#
[NEAT]
fitness_criterion = max
fitness_threshold = 10000
pop_size = 20
reset_on_extinction = True
[DefaultGenome]
# node activation options
activation_default = sigmoid
activation_mutate_rate = 0.05
activation_options = sigmoid
# node aggregation options
aggregation_default = sum
aggregation_mutate_rate = 0.05
aggregation_options = sum
# node bias options
bias_init_mean = 0.0
bias_init_stdev = 1.0
bias_max_value = 30.0
bias_min_value = -30.0
bias_mutate_power = 0.5
bias_mutate_rate = 0.7
bias_replace_rate = 0.1
# genome compatibility options
compatibility_disjoint_coefficient = 1.0
compatibility_weight_coefficient = 0.5
# connection add/remove rates
conn_add_prob = 0.5
conn_delete_prob = 0.5
# connection enable options
enabled_default = True
enabled_mutate_rate = 0.01
feed_forward = False
initial_connection = unconn nected
# node add/remove rates
node_add_prob = 0.5
node_delete_prob = 0.2
# network parameters
num_hidden = 0
num_inputs = 1120
num_outputs = 12
# node response options
response_init_mean = 1.0
response_init_stdev = 0.0
response_max_value = 30.0
response_min_value = -30.0
response_mutate_power = 0.0
response_mutate_rate = 0.0
response_replace_rate = 0.0
# connection weight options
weight_init_mean = 0.0
weight_init_stdev = 1.0
weight_max_value = 30
weight_min_value = -30
weight_mutate_power = 0.5
weight_mutate_rate = 0.8
weight_replace_rate = 0.1
[DefaultSpeciesSet]
compatibility_threshold = 205
[DefaultStagnation]
species_fitness_func = max
max_stagnation = 50
species_elitism = 0
[DefaultReproduction]
elitism = 3
survival_threshold = 0.2
the Error code from the terminal
'config-feedforward')
File "/home/gym/OPAI/lib/python3.6/site-packages/neat/config.py", line 189, in __init__
self.genome_config = genome_type.parse_config(genome_dict)
File "/home/gym/OPAI/lib/python3.6/site-packages/neat/genome.py", line 158, in parse_config
return DefaultGenomeConfig(param_dict)
File "/home/gym/OPAI/lib/python3.6/site-packages/neat/genome.py", line 72, in __init__
assert self.initial_connection in self.allowed_connectivity
AssertionError
config code from the python neat code
config = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
neat.DefaultSpeciesSet, neat.DefaultStagnation,
config-feedforward')

The issue is on the line that reads "initial_connection = unconn nected"
There's a typo. There shoudn't be a break in 'unconnected' It shouold read as follows:
"initial_connection = unconnected"

Python program doesn't write to output csv, everything else seems to work correctly

from subprocess import check_output
import csv, operator
extinction_pct = operator.itemgetter('AOT 500','AOT 675','AOT 870','AOT 936','AOT 1020')
with open('csv_export.csv') as f_csv:
reader = csv.DictReader(f_csv)
for row in reader:
with open("INPUT", 'w') as f_in:
f_in.write("&INPUT\n")
f_in.write("WLINF = 0.250\n") #lower frequency value
f_in.write("WLSUP = 4.0\n") #highest frequency value
f_in.write("WLINC = 0.5\n") #wavelength increment
f_in.write("IDAY = 289\n") #computing for a specific day
#f_in.write("ALAT = {Lat}\n".format(**row)) # for Python versions less than 3.6
f_in.write(f"ALAT = {row['Lat']}\n") #latitude of the location
#f_in.write("ALON = {Long}\n".format(**row)) # for Python versions less than 3.6
f_in.write(f"ALON = {row['Long']}\n") #longitude of the location
f_in.write("IDATM = 3\n") #atmopsheric model 2 - mid latitude summer
f_in.write("ISALB = 5\n") #surface albedo feature
f_in.write("IAER = 5\n") #boundary layer aerosol type selection - 5 - user defined spectral dependance of BLA
f_in.write("WLBAER = .500,.675,.870,.936,1.02\n") #wavelenght points for IAER
f_in.write("WBAER = 5*0.9\n") #single scattering albedo
f_in.write("GBAER = 5*0.8\n") #assymetric factor used with IAER
#f_in.write("TIME = {sama]}\n".format(**row)) # for Python versions less than 3.6
f_in.write(f"TIME = {row['sama']}\n") #Time in IST format (-5.30hr)
#f_in.write("QBAER = {}\n".format(','.join(extinction_pct(row))) # for Python versions less than 3.6
f_in.write(f"QBAER = {','.join(extinction_pct(row))}\n") #extinction efficiency percentage
f_in.write("ZOUT = 0.0,15.0\n") #TOA defining
f_in.write("/\n")
check_output('sbdart >> output1.csv',shell=True) #slarrt is the program, and ouytput.csv is the output file
This is my code, with help from #wwii
My last line, check_output csv doesnt write to my output file at all. What could be the issue?
thanks
sbdart is a program, that takes the INPUT file and outputs in the command line

Using method provided here you can try using this.
import subprocess
proc = subprocess.Popen('cmd.exe', stdin = subprocess.PIPE, stdout = subprocess.PIPE)
stdout, stderr = proc.communicate('sbdart >> output.csv')
Make sure you put the full path of sbdart or navigate to the folder having sbdart or add location of sbdart to system path
There are a bunch of other methods in the link provided

Working on Linux with python 3.5
Assume sbdart is executable and we have a file called output1.csv
sbdart looks like this for our test case:
echo $1
echo "$(cat $1)"
output1.csv is as follows:
&INPUT
WLINF = 0.250
WLSUP = 4.0
WLINC = 0.5
IDAY = 289
ALAT = {row['Lat']}
ALON = {row['Long']}
IDATM = 3
ISALB = 5
IAER = 5
WLBAER = .500,.675,.870,.936,1.02
WBAER = 5*0.9
GBAER = 5*0.8
TIME = {row['sama']}
QBAER = {','.join(extinction_pct(row))}
ZOUT = 0.0,15.0
/
>>> import subprocess
>>> subprocess.check_output(['./sbdart output1.csv'],shell=True)
b"output1.csv\n&INPUT\nWLINF = 0.250\nWLSUP = 4.0\nWLINC = 0.5\nIDAY = 289\nALAT = {row['Lat']}\nALON = {row['Long']}\nIDATM = 3\nISALB = 5\nIAER = 5\nWLBAER = .500,.675,.870,.936,1.02\nWBAER = 5*0.9\nGBAER = 5*0.8\nTIME = {row['sama']}\nQBAER = {','.join(extinction_pct(row))}\nZOUT = 0.0,15.0\n/\n"
>>>

Blender scripting - Incorrect texture add

I wrote a python script that automatically add a texture to the DAE model.
Usage: blender --background --python Script.py
This script gives an incorrect display of the texture:
import bpy
from math import pi
from mathutils import Quaternion
myscale = 0.1;
daepath = "input.dae"
bpy.ops.wm.read_factory_settings(use_empty=True)
# Import Dae
bpy.ops.wm.collada_import(filepath=daepath)
# Rotate
bpy.ops.transform.rotate(value=-pi/2, constraint_axis=(False, True, True))
bpy.ops.transform.resize(value=(myscale, myscale, myscale))
# Change to orthographic view and switch to topview
bpy.context.screen.areas[2].type = 'VIEW_3D'
rv3d = bpy.context.screen.areas[2].spaces[0].region_3d
rv3d.view_rotation = Quaternion((1.0, 0.0, 0.0, 0.0))
rv3d.view_perspective = "ORTHO"
rv3d.view_distance += 1000.0
# Unwrap
bpy.ops.object.mode_set(mode = 'EDIT')
bpy.ops.mesh.select_all(action = 'SELECT')
for oWindow in bpy.context.window_manager.windows:
oScreen = oWindow.screen
for oArea in oScreen.areas:
if oArea.type == 'VIEW_3D':
for oRegion in oArea.regions:
if oRegion.type == 'WINDOW':
override = {'window': oWindow, 'screen': oScreen, 'area': oArea, 'region': oRegion, 'scene': bpy.context.scene, 'edit_object': bpy.context.edit_object, 'active_object': bpy.context.active_object, 'selected_objects': bpy.context.selected_objects}
bpy.ops.uv.project_from_view(override , camera_bounds=False, correct_aspect=True, scale_to_bounds=True)
bpy.ops.object.mode_set(mode = 'OBJECT')
# Add Texture
imgpath = "input.jpg"
img = bpy.data.images.load(filepath=imgpath)
mat = bpy.data.materials['m0DefaultMaterial']
mat.diffuse_intensity = 1.0
mat.specular_intensity = 0.0
tex = bpy.data.textures.new("SomeName", 'IMAGE')
tex.image = img
slot = mat.texture_slots.add()
slot.texture = tex
# Save Dae
bpy.ops.wm.collada_export(filepath="output.dae")
If I divide the script into 2 parts (1 part before section # Unwrap) and run their in Blender
open app
choose scripting mode
open .py files in run
I get right result.
I was looking for all the similar problems, but I could not find the reason for this script behavior.
Tell me, please, where is the error?