1"""
2Generate a bash script to run multiple experiments.
3Make sure you generate configs from base BitstreamEvolution Directory.
4
5Origionally written by Allyn, improved and extended by Isaac.
6
7"""
8
9import os, stat
10from typing import Generator, Any, Protocol
11from dataclasses import dataclass
12
13# mkdir("data/SensitivityConfigs")
14
15# Set some defaults
16base_config_path = "data/config.ini"
17generated_configs_dir = "data/GeneratedConfigs"
18generated_bash_script_path = "data/runGeneratedConfigs.sh"
19results_output_directory = "data/GeneratedConfigsResults"
20path_best_asc_in_workspace = "./workspace/best.asc"
21path_store_best_asc = "data/previous_best.asc"
22"""This variable is where the best.asc is copied to is set command to copy_best_target_path by default"""
23
24
25# This will create the directory to put the generated configs in if it doesn't exist
26if not os.path.isdir(generated_configs_dir):
27 os.makedirs(generated_configs_dir)
28
29#Make the directory for the final results
30if not os.path.isdir(results_output_directory):
31 os.makedirs(results_output_directory)
32
33## Some Protocalls and Dataclasses to streamline operation & Provide good defaults
34
[docs]
35class CommandInfo(Protocol):
36 """A protocol specifying all data needed to invoke and run evolve on the command line."""
37 config_path:str
38 description:str
39 copy_best_asc_target_path:str|None
40 """This is the path we want to copy best.asc to once it has run successfully, None if don't want to copy it to a different location."""
41 best_asc_workspace_path:str
42 skip_next_command_if_success:bool
43 skip_next_command_if_error:bool
44 skip_next_command_if_skipped:bool
45
[docs]
46@dataclass
47class CommandData:
48 """A class that stores all data needed to invoke and run evolve on the command line."""
49 config_path:str
50 description:str
51 copy_best_asc_target_path:str|None = None
52 best_asc_workspace_path:str = path_best_asc_in_workspace
53 skip_next_command_if_success:bool = False
54 skip_next_command_if_error:bool = False
55 skip_next_command_if_skipped:bool = False
56
57
58# All {variables_in_brackets} need to be specified and formatted later as .format(variables_in_brackets="value")
59# It is fine to specify unused variables in .format(), but will error out if a value isn't included
60# You only need to include the values here that you don't want to inherit from your base config.
61
62######################### OLD SENSITIVITY SCRIPT ############################
63old_sensitivity_config_base = \
64"""[TOP-LEVEL PARAMETERS]
65base_config = {base_config_path}
66
67[FITNESS SENSITIVITY PARAMETERS]
68test_circuit = data/saved_bests/{circuit_id}.asc
69"""
70
71# return the path to the new config
72def old_sensitivity_config_generator()->Generator[CommandInfo,None,None]:
73
74 for circuit_id in range(10,510,10):
75 config_path = os.path.join(generated_configs_dir, f"{circuit_id}.ini")
76 # Create the config file using the config_base file
77 with open(config_path, "w") as config_file:
78 config_file.write(old_sensitivity_config_base.format(
79 base_config_path = base_config_path,
80 circuit_id = circuit_id
81 ))
82
83 yield CommandData(config_path=config_path,description=f"Runing sensitivity experiment on {circuit_id}.ini")
84
85
86################################## PULSE COUNT SCRIPTS ################################
87
88pulse_count_config_base = \
89"""[TOP-LEVEL PARAMETERS]
90base_config = {base_config_path}
91simulation_mode = FULLY_INTRINSIC
92
93[FITNESS PARAMETERS]
94fitness_func = {fitness_function}
95desired_freq = {desired_frequency}
96
97[GA PARAMETERS]
98population_size = {population_size}
99
100[STOPPING CONDITION PARAMETERS]
101generations = {generations}
102
103[LOGGING PARAMETERS]
104best_file = {best_asc_path}
105"""
106
[docs]
107def pulse_count_config_generator(target_pulses:list[int],
108 use_tolerant_ff:bool=True,
109 use_sensitive_ff:bool=True,
110 population_size:int=50,
111 max_generations:int=500,
112 store_best_circuit:bool=False,
113 skip_next_if_fail:bool=False,
114 skip_next_if_skipped:bool=False)->Generator[CommandInfo,None,None]:
115 """
116 Generates configs for pulse_count experiments
117
118 The order experiments will run if all fitness functions are selected is:
119 tolerant -> sensitive
120
121 Parameters
122 ----------
123 target_pulses : list[int]
124 A list of all of the target pulse counts you want to train for
125 use_tolerant_ff : bool, optional
126 If each pulse get a run using the tolerant fitness function, by default True
127 use_sensitive_ff: bool, optional
128 If each pulse get a run using the sinsitive fitness function, by default True
129 population_size: int, optional
130 This sets how many circuits are in each population, by default 50
131 max_generations: int, optional
132 Sets how many generations are allowed to run before the experiment ends, by default 500
133 store_best_circuit: bool, optional
134 If this is set the best.asc file will be copied to the data directory specified in path_store_best_asc, by default False
135 skip_next_if_fail: bool, optional
136 This will set the bash script to skip the next command it would run if this script fails, by default False
137 skip_next_if_skipped: bool, optional
138 Sets the bash script to skip the next command if this command is skipped, by default False
139
140 Yields
141 ------
142 Generator[CommandInfo,None,None]
143 The path to the output file generated.
144 """
145 def create_config(target_pulse_count:int,fitness_funciton:str,population_size:int,max_generations:int)->str:
146 # Generate the path for each pulse count
147 config_path=os.path.join(generated_configs_dir,f"{target_pulse_count}_with__{fitness_funciton}.ini")
148
149 with open(config_path, "w") as config_file:
150 config_file.write(pulse_count_config_base.format(
151 base_config_path = base_config_path,
152 fitness_function = fitness_funciton,
153 desired_frequency = target_pulse_count,
154 best_asc_path = path_best_asc_in_workspace,
155 population_size = population_size,
156 generations = max_generations
157 ))
158 return CommandData(config_path=config_path,
159 description=f"Pulse Count experiment targeting {target_pulse_count}Hz using fitness function {fitness_funciton}.",
160 copy_best_asc_target_path= path_store_best_asc if store_best_circuit else None,
161 skip_next_command_if_error=skip_next_if_fail,
162 skip_next_command_if_skipped=skip_next_if_skipped,
163 skip_next_command_if_success=False)
164
165 for target_pulse in target_pulses:
166 if use_sensitive_ff:
167 yield create_config(target_pulse,"SENSITIVE_PULSE_COUNT",population_size=population_size,max_generations=max_generations)
168 if use_tolerant_ff:
169 yield create_config(target_pulse,"TOLERANT_PULSE_COUNT",population_size=population_size,max_generations=max_generations)
170
171pulse_count_sensitivity_config_base = \
172"""[TOP-LEVEL PARAMETERS]
173base_config = {base_config_path}
174simulation_mode = INTRINSIC_SENSITIVITY
175
176[FITNESS PARAMETERS]
177fitness_func = {fitness_function}
178desired_freq = {desired_frequency}
179
180[FITNESS SENSITIVITY PARAMETERS]
181test_circuit = {test_circuit_path}
182sensitivity_trials = IGNORE
183sensitivity_time = 001:00:00:00
184reading_temp_humidity = false
185"""
186
[docs]
187def pulse_count_then_sensitivity_config_generator(target_pulses:list[int],
188 use_tolerant_ff:bool=True,
189 use_sensitive_ff:bool=True,
190 population_size:int=50,
191 max_generations:int=500)->Generator[CommandInfo,None,None]:
192 """
193 Generates configs for pulse_count experiments,
194 then follows them with the config for a sensitivity config.
195
196 The order experiments will run if all fitness functions are selected is:
197 tolerant -> sensitive
198
199 Parameters
200 ----------
201 target_pulses : list[int]
202 A list of all of the target pulse counts you want to train for
203 use_tolerant_ff : bool, optional
204 If each pulse get a run using the tolerant fitness function, by default True
205 use_sensitive_ff: bool, optional
206 If each pulse get a run using the sinsitive fitness function, by default True
207 population_size: int, optional
208 The number of circuits in each generation for pulse count, by default 50
209 max_generations: int, optional
210 The number of generations run before ending the simulation for pulse count, by default 500
211 """
212 def create_config_pair(target_pulse_count:int,fitness_funciton:str,pop_size:int,max_gens:int)->Generator[CommandInfo,None,None]:
213 # Generate a pulse_count_config_generator to get first config
214 use_sensitive = False
215 use_tolerant = False
216 match fitness_funciton:
217 case "SENSITIVE_PULSE_COUNT":
218 use_sensitive = True
219 case "TOLERANT_PULSE_COUNT":
220 use_tolerant = True
221 case bad_name:
222 raise ValueError(f"Fitness funciton name '{bad_name}' not recognised")
223
224 # This would be easier if we just pulled out the function the generator used, but I couldn't be bothered.
225 pc_gen = pulse_count_config_generator(target_pulses=[target_pulse_count],
226 use_sensitive_ff=use_sensitive,
227 use_tolerant_ff=use_tolerant,
228 population_size=pop_size,
229 max_generations=max_gens,
230 store_best_circuit=True,
231 skip_next_if_fail=True,
232 skip_next_if_skipped=True)
233 pulse_count_experiment = list(pc_gen)[0] # yield the first element only (there should only be 1)
234 yield pulse_count_experiment
235
236 # make sure we have a best.asc to act on.
237 if pulse_count_experiment.copy_best_asc_target_path is None:
238 raise ValueError("The pulse count must output to a target path for Sensitivity to test it.")
239
240 ## Generate the config for the Sensitivity run.
241 config_path=os.path.join(generated_configs_dir,f"Sensitivity_For_Pulse_Count_of_{target_pulse_count}_with_{fitness_funciton}.ini")
242
243 ## UPDATE SO TO GENERATE SENSITIVITY CONFIG 120 mins
244 with open(config_path, "w") as config_file:
245 config_file.write(pulse_count_sensitivity_config_base.format(
246 base_config_path = base_config_path,
247 fitness_function = fitness_funciton,
248 desired_frequency = target_pulse_count,
249 test_circuit_path = pulse_count_experiment.copy_best_asc_target_path # get the asc file stored by p_c_experiment
250 ))
251 yield CommandData(config_path=config_path,
252 description=f"Sensitivity Evaluation for Pulse Count experiment targeting {target_pulse_count}Hz with {fitness_funciton}.",
253 copy_best_asc_target_path= None, # No best_asc to copy out
254 skip_next_command_if_error=False,
255 skip_next_command_if_skipped=False,
256 skip_next_command_if_success=False)
257
258 for target_pulse in target_pulses:
259 if use_sensitive_ff:
260 configs = create_config_pair(target_pulse,"SENSITIVE_PULSE_COUNT",pop_size=population_size,max_gens=max_generations)
261 for config in configs:
262 yield config
263 if use_tolerant_ff:
264 configs = create_config_pair(target_pulse,"TOLERANT_PULSE_COUNT",pop_size=population_size,max_gens=max_generations)
265 for config in configs:
266 yield config
267
268
269############################ USEFUL UTILITIES ##############################################
[docs]
270def repeat(repeat_count:int, generator:Generator[Any,None,None])->Generator[Any,None,None]:
271 """
272 Repeats the outputs of the instantiated generator it is passed.
273
274 Parameters
275 ----------
276 repeat_count : int
277 number of times to duplicate the sequence
278 generator : Generator[Any,None,None]
279 Instantiated generator it duplicates
280
281 Yields
282 ------
283 Generator[Any,None,None]
284 The repeated output of the input generator
285 """
286 generator_results = list(generator)
287 for i in range(repeat_count):
288 for result in generator_results:
289 yield result
290
291
292## Select the config_generator you want to use and pass arguments
293## OPTIONS:
294# sensitivity_config_generator()
295# pulse_count_config_generator(target_pulses = [1000,10000], use_tolerant_ff = True, use_sensitive_ff = True)
296# repeat(2,pulse_count_config_generator(target_pulses = [1000, 10000], use_tolerant_ff = True, use_sensitive_ff = True))
297# pulse_count_config_generator(target_pulses = [40000,20000,20000],use_tolerant_ff=False,use_sensitive_ff=True)
298config_generator: Generator[CommandInfo,None,None] = \
299 pulse_count_then_sensitivity_config_generator(target_pulses= [1000,2000,4000],use_tolerant_ff=True, use_sensitive_ff=True,max_generations=12,population_size=53)
300## Bash File Configuration
301
302# Note that there are no spaces between variables, =, and value assigned. This is needed.
303#Bash will not recognize them as variables if there is a space in them.
304bash_head = \
305"""#!/bin/bash
306# make sure this was generated from the BitstreamEvolution folder at the base of this directory.
307
308# This variables stores the number or errors that occour
309ErrorCounter=0
310SuccessCounter=0
311SkippedCounter=0
312UserInterruptTriggered=0
313SkipNextCommand=0
314FailedCommands=''
315SkippedCommands=''
316
317
318
319#Run Commands, log if they fail.
320"""
321
322evolve_command_base = "python3 src/evolve.py -c {config_path} -d {description} -o {output_directory}"
323
324# The || only runs 2nd if left fails, && only if left succeeds
325# parenthesis is to extend command over multiple lines
326# The not means it enters the then statement if there is an error
327# the UserInterruptTriggered=$((exitCode==130)) allows us to stop other code if user interrupt occoured (set to 1 if called)
328bash_command_wrapper_logic = \
329"""
330Current_Command=$'{command}'
331if [ $UserInterruptTriggered -eq 0 ]; then
332if [ $SkipNextCommand -eq 0 ]; then
333{command}
334exitCode=$?
335if [ $exitCode -ne 0 ]; then #Error
336 UserInterruptTriggered=$((exitCode == 130))
337 ((ErrorCounter=ErrorCounter+1)) && FailedCommands+="$Current_Command"+$'\\n'
338 {action_if_failure}
339else # Success
340 ((SuccessCounter=SuccessCounter+1))
341 {action_if_success}
342fi
343else # Skipped
344 ((SkippedCounter=SkippedCounter+1)) && SkippedCommands+="$Current_Command"+$'\\n'
345 {action_if_skipped}
346fi #ctrl+c
347fi
348"""
349
350bash_tail = \
351"""
352if [ $UserInterruptTriggered -ne 0 ]; then
353#print these results if the script had a user interrupt
354echo "=================================== USER INTERRUPT RESULTS ===================================="
355echo "Canceled Commands Due to Keyboard Interrupt while running:"
356echo $"$Current_Command"
357echo ""
358echo "Commands That Failed:"
359echo "$FailedCommands"
360echo ""
361echo "Commands Skipped:"
362echo "$SkippedCommands"
363
364echo "==============================================================================================="
365echo "$ErrorCounter of $((ErrorCounter+SuccessCounter+SkippedCounter)) commands Failed, including the one not completed."
366echo "$SkippedCounter of $((ErrorCounter+SuccessCounter+SkippedCounter)) commands were Skipped"
367echo "$(({num_commands}-(ErrorCounter+SuccessCounter+SkippedCounter))) of {num_commands} were not run at all."
368
369
370exit 1
371fi
372
373# Print out the results of the Tests
374echo ""
375echo "=================================== RESULTS ===================================="
376echo ""
377echo "Commands That Failed:"
378echo "$FailedCommands"
379echo ""
380echo "Commands Skipped:"
381echo "$SkippedCommands"
382
383echo "================================================================================"
384echo "$ErrorCounter of {num_commands} commands Failed"
385echo "$SkippedCounter of {num_commands} commands were Skipped"
386
387exit 0
388
389"""
390
391with open(generated_bash_script_path, 'w') as bash_file:
392 # Invoke the bash shell for bash script
393 bash_file.write(bash_head)
394
395 command_count = 0
396 for command_data in config_generator:
397
398 # Add a call to this
399 bash_file.write(
400 bash_command_wrapper_logic.format(
401 command = evolve_command_base.format(
402 config_path = command_data.config_path,
403 description = f'"{command_data.description}"',
404 # Note that it is important that outer string uses double quotes or this messes up wrapper logic
405 output_directory = results_output_directory
406 ),
407 action_if_failure = "SkipNextCommand=1" if command_data.skip_next_command_if_error else "SkipNextCommand=0",
408 action_if_success = ("SkipNextCommand=1" if command_data.skip_next_command_if_success else "SkipNextCommand=0") +\
409 (f"\n\tcp {path_best_asc_in_workspace} {command_data.copy_best_asc_target_path}" if command_data.copy_best_asc_target_path is not None else ""),
410 action_if_skipped = "SkipNextCommand=1" if command_data.skip_next_command_if_skipped else "SkipNextCommand=0"
411 )
412 )
413
414 #count the number of commands we add
415 command_count += 1
416
417 bash_file.write(bash_tail.format(num_commands = command_count))
418
419#ensure bash script is executable if possible, but don't require permisions to do so.
420try:
421 os.chmod(generated_bash_script_path,stat.S_IREAD
422 |stat.S_IWRITE
423 |stat.S_IRWXU
424 |stat.S_IRWXG
425 |stat.S_IRWXO)
426except PermissionError:
427 print(
428 f"""
429You may need to make the bash file executable.
430Alternatively, you could run this command with sudo privilages. (sudo python3 ...)
431To do so run the following command:
432
433chmod +x {bash_file.name}
434
435"""
436 )
437
438completion_message=\
439f"""
440--------------------------------------------------------------------------
441Finished completing the bash file, and related folders.
442
443Folder containing generated partial configs: {generated_configs_dir}
444Reference base config: {base_config_path}
445Results are output to the following directory: {results_output_directory}
446
447Created the bash file at: {bash_file.name}
448---------------------------------------------------------------------------
449
450Begin the experiment by running ./{bash_file.name}
451
452---------------------------------------------------------------------------
453If you didn't run this script from the base of the BitstreamEvolution project,
454it is best you regenerate the file after you cd there and run the file from that folder, too.
455
456i.e. .../BitstreamEvolution$ python3 src/tools/generate_configs.py
457 .../BitstreamEvolution$ ./{bash_file.name}
458"""
459
460print(completion_message)
