Skip to content

Commit 6368608

Browse files
authored
Add/python parser generic (#60)
* preparing to run osu benchmarks experiments we need to have a generic parser to allow for saving the complete, raw logs. The parser is good, but we can run it after - I would rather be conservative and get the entire log vs have some bug with parsing and miss something. Signed-off-by: vsoch <vsoch@users.noreply.github.com>
1 parent e78597d commit 6368608

File tree

5 files changed

+57
-20
lines changed

5 files changed

+57
-20
lines changed

sdk/python/v1alpha1/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
1414
The versions coincide with releases on pip. Only major versions will be released as tags on Github.
1515

1616
## [0.0.x](https://github.com/converged-computing/metrics-operator/tree/main) (0.0.x)
17+
- Allow getting raw logs for any metric (without parser) (0.0.19)
1718
- Refactor of structure of Operator and addition of metrics (0.0.18)
1819
- Add wait for delete function to python parser (0.0.17)
1920
- LAMMPS python parser (0.0.16)

sdk/python/v1alpha1/metricsoperator/client.py

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,22 @@ def __init__(self, yaml_file):
2222
self.spec = utils.read_yaml(self.yaml_file)
2323
config.load_kube_config()
2424

25-
def watch(self):
25+
def watch(self, raw_logs=False, pod_prefix=None, container_name=None):
2626
"""
2727
Wait for (and yield parsed) metric logs.
2828
"""
29+
if raw_logs and not pod_prefix:
30+
raise ValueError("You must provide a pod_prefix to ask for raw logs.")
31+
2932
for metric in self.spec["spec"]["metrics"]:
30-
parser = mutils.get_metric(metric["name"])(self.spec)
33+
if raw_logs:
34+
parser = mutils.get_metric()(self.spec, container_name=container_name)
35+
else:
36+
parser = mutils.get_metric(metric["name"])(
37+
self.spec, container_name=container_name
38+
)
3139
print("Watching %s" % metric["name"])
32-
for pod, container in parser.logging_containers():
40+
for pod, container in parser.logging_containers(pod_prefix=pod_prefix):
3341
yield parser.parse(pod=pod, container=container)
3442

3543
def create(self):
@@ -65,7 +73,7 @@ def namespace(self):
6573
def name(self):
6674
return self.spec["metadata"]["name"]
6775

68-
def delete(self):
76+
def delete(self, pod_prefix=None):
6977
"""
7078
Delete the associated YAML file.
7179
"""
@@ -77,14 +85,14 @@ def delete(self):
7785
plural=self.plural,
7886
name=self.name,
7987
)
80-
self.wait_for_delete()
88+
self.wait_for_delete(pod_prefix)
8189
return result
8290

83-
def wait_for_delete(self):
91+
def wait_for_delete(self, pod_prefix=None):
8492
"""
8593
Wait for pods to be gone (deleted)
8694
"""
8795
for metric in self.spec["spec"]["metrics"]:
8896
parser = mutils.get_metric(metric["name"])(self.spec)
8997
print("Watching %s for deletion" % metric["name"])
90-
parser.wait_for_delete()
98+
parser.wait_for_delete(pod_prefix=pod_prefix)

sdk/python/v1alpha1/metricsoperator/metrics/__init__.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
33

44
import metricsoperator.metrics.app as apps
5+
import metricsoperator.metrics.base as base
56
import metricsoperator.metrics.network as network
67
import metricsoperator.metrics.perf as perf
78
import metricsoperator.metrics.storage as storage
@@ -17,11 +18,12 @@
1718
}
1819

1920

20-
def get_metric(name):
21+
def get_metric(name=None):
2122
"""
2223
Get a named metric parser.
2324
"""
2425
metric = metrics.get(name)
26+
# If we don't have a matching metric, return base (for raw logs)
2527
if not metric:
26-
raise ValueError(f"Metric {name} does not have a known parser")
28+
return base.MetricBase
2729
return metric

sdk/python/v1alpha1/metricsoperator/metrics/base.py

Lines changed: 36 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ class MetricBase:
1313
collection_end = "METRICS OPERATOR COLLECTION END"
1414
metadata_start = "METADATA START"
1515
metadata_end = "METADATA END"
16+
container_name = None
1617

1718
def __init__(self, spec=None, **kwargs):
1819
"""
@@ -23,6 +24,10 @@ def __init__(self, spec=None, **kwargs):
2324
self.spec = spec
2425
self._core_v1 = kwargs.get("core_v1_api")
2526

27+
# If we don't have a default container name...
28+
if not self.container_name:
29+
self.container_name = kwargs.get("container_name") or "launcher"
30+
2631
# Load kubeconfig on Metricbase init only
2732
if self.spec is not None:
2833
config.load_kube_config()
@@ -55,6 +60,14 @@ def parse(self, pod, container):
5560
)
5661
return self.parse_log(lines)
5762

63+
def parse_log(self, lines):
64+
"""
65+
If the parser doesn't have anything, just return the lines
66+
"""
67+
# Get the log metadata, split lines by newline so not so hefty a log!
68+
metadata = self.get_log_metadata(lines)
69+
return {"data": lines.split("\n"), "metadata": metadata, "spec": self.spec}
70+
5871
@property
5972
def core_v1(self):
6073
"""
@@ -69,12 +82,14 @@ def core_v1(self):
6982
self._core_v1 = core_v1_api.CoreV1Api()
7083
return self._core_v1
7184

72-
def logging_containers(self, namespace=None, states=None, retry_seconds=5):
85+
def logging_containers(
86+
self, namespace=None, states=None, retry_seconds=5, pod_prefix=None
87+
):
7388
"""
7489
Return list of containers intended to get logs from
7590
"""
7691
containers = []
77-
pods = self.wait(namespace, states, retry_seconds)
92+
pods = self.wait(namespace, states, retry_seconds, pod_prefix=pod_prefix)
7893
container_name = getattr(self, "container_name", self.container)
7994
print(f"Looking for container name {container_name}...")
8095
for pod in pods.items:
@@ -90,17 +105,27 @@ def logging_containers(self, namespace=None, states=None, retry_seconds=5):
90105
)
91106
return containers
92107

93-
def wait(self, namespace=None, states=None, retry_seconds=5):
108+
def get_pod_prefix(self, pod_prefix=None):
109+
"""
110+
Return the default or a custom pod prefix.
111+
"""
112+
pod_prefix = pod_prefix or getattr(self, "pod_prefix", None)
113+
if not pod_prefix:
114+
raise ValueError("A pod prefix 'pod_prefix' is required to wait for pods.")
115+
return pod_prefix
116+
117+
def wait(self, namespace=None, states=None, retry_seconds=5, pod_prefix=None):
94118
"""
95119
Wait for one or more pods of interest to be done.
96120
97121
This assumes creation or a consistent size of pod getting to a
98122
particular state. If looking for Termination -> gone, use
99123
wait_for_delete.
100124
"""
125+
pod_prefix = self.get_pod_prefix(pod_prefix)
101126
namespace = namespace or self.namespace
102-
print(f"Looking for prefix {self.pod_prefix} in namespace {namespace}")
103-
pod_list = self.get_pods(namespace, self.pod_prefix)
127+
print(f"Looking for prefix {pod_prefix} in namespace {namespace}")
128+
pod_list = self.get_pods(namespace, pod_prefix)
104129
size = len(pod_list.items)
105130

106131
# We only want logs when they are completed
@@ -111,7 +136,7 @@ def wait(self, namespace=None, states=None, retry_seconds=5):
111136
ready = set()
112137
while len(ready) != size:
113138
print(f"{len(ready)} pods are ready, out of {size}")
114-
pod_list = self.get_pods(name=self.pod_prefix, namespace=namespace)
139+
pod_list = self.get_pods(name=pod_prefix, namespace=namespace)
115140

116141
for pod in pod_list.items:
117142
print(f"{pod.metadata.name} is in phase {pod.status.phase}")
@@ -126,16 +151,17 @@ def wait(self, namespace=None, states=None, retry_seconds=5):
126151
print(f'All pods are in states "{states}"')
127152
return pod_list
128153

129-
def wait_for_delete(self, namespace=None, retry_seconds=5):
154+
def wait_for_delete(self, namespace=None, retry_seconds=5, pod_prefix=None):
130155
"""
131156
Wait for one or more pods of interest to be gone
132157
"""
158+
pod_prefix = self.get_pod_prefix(pod_prefix)
133159
namespace = namespace or self.namespace
134-
print(f"Looking for prefix {self.pod_prefix} in namespace {namespace}")
135-
pod_list = self.get_pods(namespace, name=self.pod_prefix)
160+
print(f"Looking for prefix {pod_prefix} in namespace {namespace}")
161+
pod_list = self.get_pods(namespace, name=pod_prefix)
136162
while len(pod_list.items) != 0:
137163
print(f"{len(pod_list.items)} pods exist, waiting for termination.")
138-
pod_list = self.get_pods(name=self.pod_prefix, namespace=namespace)
164+
pod_list = self.get_pods(name=pod_prefix, namespace=namespace)
139165
time.sleep(retry_seconds)
140166
print("All pods are terminated.")
141167

sdk/python/v1alpha1/setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
if __name__ == "__main__":
3131
setup(
3232
name="metricsoperator",
33-
version="0.0.18",
33+
version="0.0.19",
3434
author="Vanessasaurus",
3535
author_email="vsoch@users.noreply.github.com",
3636
maintainer="Vanessasaurus",

0 commit comments

Comments
 (0)