Skip to content

Commit 4566089

Browse files
jeffng-orjeffng-or
committed
Adding gen_at_report plus unit tests
lint fixes Signed-off-by: Jeff Ng <jeffng@precisioninno.com>
1 parent 804812e commit 4566089

File tree

17 files changed

+1358
-2
lines changed

17 files changed

+1358
-2
lines changed

‎tools/AutoTuner/src/autotuner/utils/__init__.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# To get relative imports to work
2+
import os, sys
3+
4+
sys.path.append(os.path.dirname(os.path.realpath(__file__)))

‎tools/AutoTuner/src/autotuner/utils/at_dir_extractor.py

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
#!/usr/bin/env python3
2+
#############################################################################
3+
##
4+
## Copyright (c) 2024, Precision Innovations Inc.
5+
## All rights reserved.
6+
##
7+
## BSD 3-Clause License
8+
##
9+
## Redistribution and use in source and binary forms, with or without
10+
## modification, are permitted provided that the following conditions are met:
11+
##
12+
## * Redistributions of source code must retain the above copyright notice, this
13+
## list of conditions and the following disclaimer.
14+
##
15+
## * Redistributions in binary form must reproduce the above copyright notice,
16+
## this list of conditions and the following disclaimer in the documentation
17+
## and/or other materials provided with the distribution.
18+
##
19+
## * Neither the name of the copyright holder nor the names of its
20+
## contributors may be used to endorse or promote products derived from
21+
## this software without specific prior written permission.
22+
##
23+
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24+
## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25+
## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26+
## ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
27+
## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28+
## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29+
## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30+
## INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31+
## CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32+
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33+
## POSSIBILITY OF SUCH DAMAGE.
34+
###############################################################################
35+
36+
import os
37+
import sys
38+
import time
39+
import argparse
40+
import json
41+
from at_extractor_base import ATExtractorBase
42+
43+
44+
class ATDirExtractor(ATExtractorBase):
45+
def __init__(self, obj, completion_cbk, result_cbk):
46+
"""Initializer"""
47+
48+
super().__init__(obj, completion_cbk, result_cbk)
49+
50+
def extract_dir(self, dir_name):
51+
"""Extracts the data from the log directory"""
52+
53+
for trial_name in os.listdir(dir_name):
54+
if trial_name.startswith("variant-AutoTunerBase-") and trial_name.endswith(
55+
"-ray"
56+
):
57+
self._extract_dir(trial_name, os.path.join(dir_name, trial_name))
58+
59+
def _extract_dir(self, trial_name, dir_name):
60+
"""Extracts the data from the result.json file"""
61+
62+
results_file = os.path.join(dir_name, "result.json")
63+
if os.path.exists(results_file):
64+
with open(results_file, "r") as fh:
65+
data = json.load(fh)
66+
metrics = data.copy()
67+
metrics["metric"] = round(metrics["metric"], 1)
68+
self._result_cbk(self._obj, trial_name, metrics)
69+
run_time = time.strftime(
70+
"%H:%M:%S", time.gmtime(data["time_this_iter_s"])
71+
)
72+
self._completion_cbk(self._obj, trial_name, data["timestamp"], run_time)
73+
74+
@staticmethod
75+
def main():
76+
"""Standalone test driver"""
77+
parser = argparse.ArgumentParser(prog="at_dir_extractor.py")
78+
parser.add_argument(
79+
"-d",
80+
"--dir_name",
81+
help="ORFS log directory for AutoTuner run",
82+
required=True,
83+
)
84+
args = parser.parse_args()
85+
rep = ATDirExtractor(
86+
None,
87+
ATExtractorBase.sample_completion_callback,
88+
ATExtractorBase.sample_results_callback,
89+
)
90+
rep.extract_dir(args.dir_name)
91+
92+
93+
if __name__ == "__main__":
94+
ATDirExtractor.main()
95+
sys.exit(0)

‎tools/AutoTuner/src/autotuner/utils/at_extractor_base.py

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#!/usr/bin/env python3
2+
#############################################################################
3+
##
4+
## Copyright (c) 2024, Precision Innovations Inc.
5+
## All rights reserved.
6+
##
7+
## BSD 3-Clause License
8+
##
9+
## Redistribution and use in source and binary forms, with or without
10+
## modification, are permitted provided that the following conditions are met:
11+
##
12+
## * Redistributions of source code must retain the above copyright notice, this
13+
## list of conditions and the following disclaimer.
14+
##
15+
## * Redistributions in binary form must reproduce the above copyright notice,
16+
## this list of conditions and the following disclaimer in the documentation
17+
## and/or other materials provided with the distribution.
18+
##
19+
## * Neither the name of the copyright holder nor the names of its
20+
## contributors may be used to endorse or promote products derived from
21+
## this software without specific prior written permission.
22+
##
23+
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24+
## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25+
## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26+
## ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
27+
## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28+
## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29+
## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30+
## INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31+
## CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32+
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33+
## POSSIBILITY OF SUCH DAMAGE.
34+
###############################################################################
35+
36+
from datetime import datetime
37+
38+
39+
class ATExtractorBase:
40+
"""Base class for AutoTuner extractors"""
41+
42+
def __init__(self, obj, completion_cbk, result_cbk):
43+
"""Registers context object, completion callback and results callback"""
44+
45+
self._completion_cbk = completion_cbk
46+
self._result_cbk = result_cbk
47+
self._obj = obj
48+
49+
@staticmethod
50+
def get_completion_time(timestamp):
51+
"""Returns a string for the completion time timestamp"""
52+
datetime_obj = datetime.fromtimestamp(timestamp)
53+
datetime_str = datetime_obj.strftime("%Y-%m-%d %H:%M:%S")
54+
return datetime_str
55+
56+
@staticmethod
57+
def sample_completion_callback(obj, trial_name, completion_timestamp, run_time):
58+
"""Sample completion callback"""
59+
completion_time = ATExtractorBase.get_completion_time(completion_timestamp)
60+
print(
61+
f"Trial {trial_name} finished at {completion_time} with run time {run_time}"
62+
)
63+
64+
@staticmethod
65+
def sample_results_callback(obj, trial_name, metrics):
66+
"""Sample results callback"""
67+
print(f"Trial {trial_name} metrics {metrics}")

‎tools/AutoTuner/src/autotuner/utils/at_log_extractor.py

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
#!/usr/bin/env python3
2+
#############################################################################
3+
##
4+
## Copyright (c) 2024, Precision Innovations Inc.
5+
## All rights reserved.
6+
##
7+
## BSD 3-Clause License
8+
##
9+
## Redistribution and use in source and binary forms, with or without
10+
## modification, are permitted provided that the following conditions are met:
11+
##
12+
## * Redistributions of source code must retain the above copyright notice, this
13+
## list of conditions and the following disclaimer.
14+
##
15+
## * Redistributions in binary form must reproduce the above copyright notice,
16+
## this list of conditions and the following disclaimer in the documentation
17+
## and/or other materials provided with the distribution.
18+
##
19+
## * Neither the name of the copyright holder nor the names of its
20+
## contributors may be used to endorse or promote products derived from
21+
## this software without specific prior written permission.
22+
##
23+
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24+
## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25+
## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26+
## ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
27+
## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28+
## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29+
## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30+
## INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31+
## CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32+
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33+
## POSSIBILITY OF SUCH DAMAGE.
34+
###############################################################################
35+
36+
import re
37+
import sys
38+
import argparse
39+
from datetime import datetime
40+
from at_extractor_base import ATExtractorBase
41+
42+
43+
class ATLogExtractor(ATExtractorBase):
44+
"""Class to extract AutoTuner results from the AutoTuner output"""
45+
46+
def __init__(self, obj, completion_cbk, result_cbk):
47+
"""Initializes RE's to parse log file"""
48+
49+
super().__init__(obj, completion_cbk, result_cbk)
50+
self._trial_completed_re = re.compile(
51+
"Trial\s+(\S+)\s+completed\s+after\s+\d+\s+iterations\s+at\s+(\S+)\s+(\S+)\.\s+Total\s+running\s+time\:\s+((\d+)h\s+)?((\d+)min\s+)?(\d+)s"
52+
)
53+
self._trial_result_table_start_re = re.compile("Trial\s+(\S+)\s+result")
54+
self._trial_result_table_metric_re = re.compile("\s+(\w+)\s+(-?\d+(\.\d+)?)\s+")
55+
56+
def extract_file(self, file_name):
57+
"""Opens and reads the log file to populate the trial dictionary"""
58+
59+
with open(file_name, "r") as fh:
60+
self._extract(fh)
61+
62+
### Private API
63+
64+
def _get_timestamp(self, date_str, time_str):
65+
"""Converts the date and time strings to a timestamp"""
66+
67+
date_time_str = f"{date_str} {time_str}"
68+
date_time_obj = datetime.strptime(date_time_str, "%Y-%m-%d %H:%M:%S")
69+
return date_time_obj.timestamp()
70+
71+
def _read_result_table(self, fh, trial_name):
72+
"""Reads the metrics from the results table and returns it in a dict"""
73+
74+
metrics = {}
75+
line = fh.readline()
76+
while line:
77+
if line == "\n":
78+
return metrics
79+
result = self._trial_result_table_metric_re.search(line)
80+
if result:
81+
metric_name = result.group(1)
82+
metric_value = result.group(2)
83+
metrics[metric_name] = metric_value
84+
line = fh.readline()
85+
86+
def _add_completed_trial(self, result):
87+
"""Extracts results from RE and adds data to trial dictionary"""
88+
89+
trial_name = result.group(1)
90+
completion_date = result.group(2)
91+
completion_time = result.group(3)
92+
run_time = self._get_runtime(result)
93+
completion_timestamp = self._get_timestamp(completion_date, completion_time)
94+
self._completion_cbk(self._obj, trial_name, completion_timestamp, run_time)
95+
96+
def _add_result(self, fh, trial_name):
97+
"""Extracts results from table and calls result callback"""
98+
99+
metrics = self._read_result_table(fh, trial_name)
100+
self._result_cbk(self._obj, trial_name, metrics)
101+
102+
def _extract(self, fh):
103+
"""Parses the stream and extracts the data"""
104+
105+
for line in fh:
106+
result = self._trial_completed_re.match(line)
107+
if result:
108+
self._add_completed_trial(result)
109+
else:
110+
result = self._trial_result_table_start_re.search(line)
111+
if result:
112+
trial_name = result.group(1)
113+
self._add_result(fh, trial_name)
114+
115+
def _get_runtime(self, result):
116+
"""
117+
Returns the runtime string in h:mm:ss format, based on the RE match
118+
input
119+
"""
120+
121+
runtime_hr = runtime_min = runtime_sec = 0
122+
# 4 is the optional wrapper around hours
123+
# 5 is the hours
124+
if result.group(4):
125+
runtime_hr = int(result.group(5))
126+
# 6 is the optional wrapper around mins
127+
# 7 is the minutes
128+
if result.group(6):
129+
runtime_min = int(result.group(7))
130+
runtime_sec = int(result.group(8))
131+
run_time = f"{runtime_hr}:{runtime_min:02}:{runtime_sec:02}"
132+
return run_time
133+
134+
@staticmethod
135+
def main(): # pragma: nocover
136+
"""Standalone test driver"""
137+
138+
parser = argparse.ArgumentParser(prog="at_log_extractor.py")
139+
parser.add_argument(
140+
"-i", "--input_file", help="log file from AutoTuner", required=True
141+
)
142+
args = parser.parse_args()
143+
rep = ATLogExtractor(
144+
None,
145+
ATExtractorBase.sample_completion_callback,
146+
ATExtractorBase.sample_results_callback,
147+
)
148+
rep.extract_file(args.input_file)
149+
150+
151+
if __name__ == "__main__": # pragma: nocover
152+
ATLogExtractor.main()
153+
sys.exit(0)

0 commit comments

Comments
 (0)