1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 from haizea.core.manager import Manager
20 from haizea.common.utils import generate_config_name, unpickle
21 from haizea.core.configfile import HaizeaConfig, HaizeaMultiConfig
22 from haizea.core.accounting import AccountingDataCollection
23 from haizea.common.config import ConfigException
24 from haizea.cli.optionparser import Option
25 from haizea.cli import Command
26 from mx.DateTime import TimeDelta
27 import haizea.common.defaults as defaults
28 import sys
29 import os
30 import errno
31 import signal
32 from time import sleep
33
34 try:
35 import xml.etree.ElementTree as ET
36 except ImportError:
37
38 import elementtree.ElementTree as ET
39
40
42 """
43 This is the main Haizea command. By default, it will start Haizea as a daemon, which
44 can receive requests via RPC or interact with other components such as OpenNebula. It can
45 also start as a foreground process, and write all log messages to the console. All
46 Haizea options are specified through the configuration file."""
47
48 name = "haizea"
49
51 Command.__init__(self, argv)
52
53 self.optparser.add_option(Option("-c", "--conf", action="store", type="string", dest="conf",
54 help = """
55 The location of the Haizea configuration file. If not
56 specified, Haizea will first look for it in
57 /etc/haizea/haizea.conf and then in ~/.haizea/haizea.conf.
58 """))
59 self.optparser.add_option(Option("-f", "--fg", action="store_true", dest="foreground",
60 help = """
61 Runs Haizea in the foreground.
62 """))
63 self.optparser.add_option(Option("--stop", action="store_true", dest="stop",
64 help = """
65 Stops the Haizea daemon.
66 """))
67
69 self.parse_options()
70
71 pidfile = defaults.DAEMON_PIDFILE
72
73 if self.opt.stop == None:
74
75
76
77 if os.path.exists(pidfile):
78 pf = file(pidfile,'r')
79 pid = int(pf.read().strip())
80 pf.close()
81
82 try:
83 os.kill(pid, signal.SIG_DFL)
84 except OSError, (err, msg):
85 if err == errno.ESRCH:
86
87 os.remove(pidfile)
88 else:
89 msg = "Unexpected error when checking pid file '%s'.\n%s\n" %(pidfile, msg)
90 sys.stderr.write(msg)
91 sys.exit(1)
92 else:
93 msg = "Haizea seems to be already running (pid %i)\n" % pid
94 sys.stderr.write(msg)
95 sys.exit(1)
96
97 try:
98 configfile=self.opt.conf
99 if configfile == None:
100
101 for loc in defaults.CONFIG_LOCATIONS:
102 if os.path.exists(loc):
103 config = HaizeaConfig.from_file(loc)
104 break
105 else:
106 print >> sys.stdout, "No configuration file specified, and none found at default locations."
107 print >> sys.stdout, "Make sure a config file exists at:\n -> %s" % "\n -> ".join(defaults.CONFIG_LOCATIONS)
108 print >> sys.stdout, "Or specify a configuration file with the --conf option."
109 exit(1)
110 else:
111 config = HaizeaConfig.from_file(configfile)
112 except ConfigException, msg:
113 print >> sys.stderr, "Error in configuration file:"
114 print >> sys.stderr, msg
115 exit(1)
116
117 daemon = not self.opt.foreground
118
119 manager = Manager(config, daemon, pidfile)
120
121 manager.start()
122 elif self.opt.stop:
123
124 try:
125 pf = file(pidfile,'r')
126 pid = int(pf.read().strip())
127 pf.close()
128 except IOError:
129 msg = "Could not stop, pid file '%s' missing.\n"
130 sys.stderr.write(msg % pidfile)
131 sys.exit(1)
132 try:
133 while 1:
134 os.kill(pid, signal.SIGTERM)
135 sleep(1)
136 except OSError, err:
137 err = str(err)
138 if err.find("No such process") > 0:
139 os.remove(pidfile)
140 else:
141 print str(err)
142 sys.exit(1)
143
145 """
146 Takes an Haizea multiconfiguration file and generates the individual
147 configuration files. See the Haizea manual for more details on multiconfiguration
148 files."""
149
150 name = "haizea-generate-configs"
151
153 Command.__init__(self, argv)
154
155 self.optparser.add_option(Option("-c", "--conf", action="store", type="string", dest="conf", required=True,
156 help = """
157 Multiconfiguration file.
158 """))
159 self.optparser.add_option(Option("-d", "--dir", action="store", type="string", dest="dir", required=True,
160 help = """
161 Directory where the individual configuration files
162 must be created.
163 """))
164
166 self.parse_options()
167
168 configfile=self.opt.conf
169 multiconfig = HaizeaMultiConfig.from_file(configfile)
170
171 etcdir = self.opt.dir
172
173 configs = multiconfig.get_configs()
174
175 etcdir = os.path.abspath(etcdir)
176 if not os.path.exists(etcdir):
177 os.makedirs(etcdir)
178
179 for c in configs:
180 profile = c.get_attr("profile")
181 tracefile = c.get("tracefile")
182 injfile = c.get("injectionfile")
183 configname = generate_config_name(profile, tracefile, injfile)
184 configfile = etcdir + "/%s.conf" % configname
185 fc = open(configfile, "w")
186 c.config.write(fc)
187 fc.close()
188
190 """
191 Generates a script, based on a script template, to run all the individual
192 configuration files generated by haizea-generate-configs. This command
193 requires Mako Templates for Python (http://www.makotemplates.org/)."""
194
195 name = "haizea-generate-scripts"
196
198 Command.__init__(self, argv)
199
200 self.optparser.add_option(Option("-c", "--conf", action="store", type="string", dest="conf", required=True,
201 help = """
202 Multiconfiguration file used in haizea-generate-configs.
203 """))
204 self.optparser.add_option(Option("-d", "--confdir", action="store", type="string", dest="confdir", required=True,
205 help = """
206 Directory containing the individual configuration files.
207 """))
208 self.optparser.add_option(Option("-t", "--template", action="store", type="string", dest="template", required=True,
209 help = """
210 Script template (sample templates are included in /usr/share/haizea/etc)
211 """))
212 self.optparser.add_option(Option("-m", "--only-missing", action="store_true", dest="onlymissing",
213 help = """
214 If specified, the generated script will only run the configurations
215 that have not already produced a datafile. This is useful when some simulations
216 fail, and you don't want to have to rerun them all.
217 """))
218
220 self.parse_options()
221
222 configfile=self.opt.conf
223 multiconfig = HaizeaMultiConfig.from_file(configfile)
224
225 try:
226 from mako.template import Template
227 except Exception, e:
228 print "You need Mako Templates for Python to run this command."
229 print "You can download them at http://www.makotemplates.org/"
230 exit(1)
231
232 configs = multiconfig.get_configs()
233
234 etcdir = os.path.abspath(self.opt.confdir)
235 if not os.path.exists(etcdir):
236 os.makedirs(etcdir)
237
238 templatedata = []
239 for c in configs:
240 profile = c.get_attr("profile")
241 tracefile = c.get("tracefile")
242 injfile = c.get("injectionfile")
243 datafile = c.get("datafile")
244 configname = generate_config_name(profile, tracefile, injfile)
245 if not self.opt.onlymissing or not os.path.exists(datafile):
246 configfile = etcdir + "/%s.conf" % configname
247 templatedata.append((configname, configfile))
248
249 template = Template(filename=self.opt.template)
250 print template.render(configs=templatedata, etcdir=etcdir)
251
252
254 """
255 Converts Haizea datafiles into another (easier to process) format.
256 """
257
258 name = "haizea-convert-data"
259
261 Command.__init__(self, argv)
262
263 self.optparser.add_option(Option("-t", "--type", action="store", dest="type",
264 choices = ["per-run", "per-lease", "counter"],
265 help = """
266 Type of data to produce.
267 """))
268 self.optparser.add_option(Option("-c", "--counter", action="store", dest="counter",
269 help = """
270 Counter to print out when using '--type counter'.
271 """))
272 self.optparser.add_option(Option("-f", "--format", action="store", type="string", dest="format",
273 help = """
274 Output format. Currently supported: csv
275 """))
276 self.optparser.add_option(Option("-l", "--list-counters", action="store_true", dest="list_counters",
277 help = """
278 If specified, the command will just print out the names of counters
279 stored in the data file and then exit, regardless of other parameters.
280 """))
281
283 self.parse_options()
284
285 datafiles=self.args[1:]
286 if len(datafiles) == 0:
287 print "Please specify at least one datafile to convert"
288 exit(1)
289
290 datafile1 = unpickle(datafiles[0])
291
292 counter_names = datafile1.counters.keys()
293 attr_names = datafile1.attrs.keys()
294 lease_stats_names = datafile1.lease_stats_names
295 stats_names = datafile1.stats_names
296
297 if self.opt.list_counters:
298 for counter in counter_names:
299 print counter
300 exit(0)
301
302 if self.opt.type == "per-run":
303 header_fields = attr_names + stats_names
304 elif self.opt.type == "per-lease":
305 header_fields = attr_names + ["lease_id"] + lease_stats_names
306 elif self.opt.type == "counter":
307 counter = self.opt.counter
308 if not datafile1.counters.has_key(counter):
309 print "The specified datafile does not have a counter called '%s'" % counter
310 exit(1)
311 header_fields = attr_names + ["time", "value"]
312 if datafile1.counter_avg_type[counter] != AccountingDataCollection.AVERAGE_NONE:
313 header_fields.append("average")
314
315 header = ",".join(header_fields)
316
317 print header
318
319 for datafile in datafiles:
320 data = unpickle(datafile)
321
322 attrs = [data.attrs[attr_name] for attr_name in attr_names]
323
324 if self.opt.type == "per-run":
325 fields = attrs + [`data.stats[stats_name]` for stats_name in stats_names]
326 print ",".join(fields)
327 elif self.opt.type == "per-lease":
328 leases = data.lease_stats
329 for lease_id, lease_stat in leases.items():
330 fields = attrs + [`lease_id`] + [`lease_stat.get(lease_stat_name,"")` for lease_stat_name in lease_stats_names]
331 print ",".join(fields)
332 elif self.opt.type == "counter":
333 for (time, lease_id, value, avg) in data.counters[counter]:
334 fields = attrs + [`time`, `value`]
335 if data.counter_avg_type[counter] != AccountingDataCollection.AVERAGE_NONE:
336 fields.append(`avg`)
337 print ",".join(fields)
338
339
340
342 """
343 Converts old Haizea LWF file into new XML-based LWF format
344 """
345
346 name = "haizea-lwf2xml"
347
349 Command.__init__(self, argv)
350
351 self.optparser.add_option(Option("-i", "--in", action="store", type="string", dest="inf",
352 help = """
353 Input file
354 """))
355 self.optparser.add_option(Option("-o", "--out", action="store", type="string", dest="outf",
356 help = """
357 Output file
358 """))
359
361 self.parse_options()
362
363 infile = self.opt.inf
364 outfile = self.opt.outf
365
366 root = ET.Element("lease-workload")
367 root.set("name", infile)
368 description = ET.SubElement(root, "description")
369 time = TimeDelta(seconds=0)
370 lease_id = 1
371 requests = ET.SubElement(root, "lease-requests")
372
373
374 infile = open(infile, "r")
375 for line in infile:
376 if line[0]!='#' and len(line.strip()) != 0:
377 fields = line.split()
378 submit_time = int(fields[0])
379 start_time = int(fields[1])
380 duration = int(fields[2])
381 real_duration = int(fields[3])
382 num_nodes = int(fields[4])
383 cpu = int(fields[5])
384 mem = int(fields[6])
385 disk = int(fields[7])
386 vm_image = fields[8]
387 vm_imagesize = int(fields[9])
388
389
390
391 lease_request = ET.SubElement(requests, "lease-request")
392 lease_request.set("arrival", str(TimeDelta(seconds=submit_time)))
393 if real_duration != duration:
394 realduration = ET.SubElement(lease_request, "realduration")
395 realduration.set("time", str(TimeDelta(seconds=real_duration)))
396
397 lease = ET.SubElement(lease_request, "lease")
398 lease.set("id", `lease_id`)
399
400
401 nodes = ET.SubElement(lease, "nodes")
402 node_set = ET.SubElement(nodes, "node-set")
403 node_set.set("numnodes", `num_nodes`)
404 res = ET.SubElement(node_set, "res")
405 res.set("type", "CPU")
406 if cpu == 1:
407 res.set("amount", "100")
408 else:
409 pass
410 res = ET.SubElement(node_set, "res")
411 res.set("type", "Memory")
412 res.set("amount", `mem`)
413
414 start = ET.SubElement(lease, "start")
415 if start_time == -1:
416 lease.set("preemptible", "true")
417 else:
418 lease.set("preemptible", "false")
419 exact = ET.SubElement(start, "exact")
420 exact.set("time", str(TimeDelta(seconds=start_time)))
421
422 duration_elem = ET.SubElement(lease, "duration")
423 duration_elem.set("time", str(TimeDelta(seconds=duration)))
424
425 software = ET.SubElement(lease, "software")
426 diskimage = ET.SubElement(software, "disk-image")
427 diskimage.set("id", vm_image)
428 diskimage.set("size", `vm_imagesize`)
429
430
431 lease_id += 1
432 tree = ET.ElementTree(root)
433 print ET.tostring(root)
434
435