1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29 package uk.ac.rdg.resc.jstyx.gridservice.client;
30
31 import java.io.IOException;
32 import java.io.PrintStream;
33 import java.io.PrintWriter;
34 import java.io.OutputStream;
35 import java.io.FileOutputStream;
36 import java.io.FileNotFoundException;
37 import java.io.File;
38
39 import java.util.Hashtable;
40 import java.util.Iterator;
41 import java.util.Vector;
42 import java.util.Enumeration;
43
44 import java.net.UnknownHostException;
45
46 import org.apache.log4j.Logger;
47
48 import com.martiansoftware.jsap.JSAP;
49 import com.martiansoftware.jsap.JSAPException;
50 import com.martiansoftware.jsap.JSAPResult;
51 import com.martiansoftware.jsap.Switch;
52 import com.martiansoftware.jsap.FlaggedOption;
53 import com.martiansoftware.jsap.StringParser;
54 import com.martiansoftware.jsap.ParseException;
55
56 import uk.ac.rdg.resc.jstyx.client.StyxConnection;
57 import uk.ac.rdg.resc.jstyx.client.StyxConnectionListener;
58
59 import uk.ac.rdg.resc.jstyx.StyxException;
60
61 import uk.ac.rdg.resc.jstyx.gridservice.config.SGSConfig;
62 import uk.ac.rdg.resc.jstyx.gridservice.config.SGSParam;
63 import uk.ac.rdg.resc.jstyx.gridservice.config.SGSInput;
64 import uk.ac.rdg.resc.jstyx.gridservice.config.SGSOutput;
65
66 /***
67 * Simple program that logs on to an SGS server, creates a new service instance,
68 * runs it and redirects the output streams to the console and local files
69 *
70 * @author Jon Blower
71 * $Revision: 609 $
72 * $Date: 2006-03-31 18:09:42 +0100 (Fri, 31 Mar 2006) $
73 * $Log: SGSRun.java,v $
74 * Revision 1.20 2006/02/22 08:52:57 jonblower
75 * Added debug code and support for setting service lifetime
76 *
77 * Revision 1.19 2006/02/20 17:35:01 jonblower
78 * Implemented correct handling of output files/streams (not fully tested yet)
79 *
80 * Revision 1.18 2006/02/17 17:34:44 jonblower
81 * Implemented (but didn't test) proper handling of output files
82 *
83 * Revision 1.17 2006/02/17 09:27:50 jonblower
84 * Working towards handling output files properly
85 *
86 * Revision 1.16 2006/02/16 17:34:16 jonblower
87 * Working towards handling output files and references thereto
88 *
89 * Revision 1.15 2006/01/05 16:06:34 jonblower
90 * SGS clients now deal with possibility that client could be created on a different server
91 *
92 * Revision 1.14 2006/01/04 11:24:57 jonblower
93 * Implemented time directory in the SGS instance namespace
94 *
95 * Revision 1.13 2005/12/09 18:41:56 jonblower
96 * Continuing to simplify client interface to SGS instances
97 *
98 * Revision 1.12 2005/12/07 17:53:31 jonblower
99 * Added type to SGSParam (STRING, INPUT_FILE and OUTPUT_FILE)
100 *
101 * Revision 1.11 2005/12/07 08:56:32 jonblower
102 * Refactoring SGS client code
103 *
104 * Revision 1.10 2005/12/01 17:17:07 jonblower
105 * Simplifying client interface to SGS instances
106 *
107 * Revision 1.9 2005/12/01 08:29:47 jonblower
108 * Refactored XML config handling to simplify clients
109 *
110 * Revision 1.8 2005/11/28 17:20:18 jonblower
111 * Fixed bug with not exiting cleanly when error occurs
112 *
113 * Revision 1.7 2005/11/14 21:31:54 jonblower
114 * Got SGSRun working for SC2005 demo
115 *
116 * Revision 1.6 2005/11/11 21:57:21 jonblower
117 * Implemented passing of URLs to input files
118 *
119 * Revision 1.5 2005/11/10 19:50:43 jonblower
120 * Added code to handle output files
121 *
122 * Revision 1.4 2005/11/09 18:00:24 jonblower
123 * Implemented automatic uploading of input files
124 *
125 * Revision 1.3 2005/10/18 14:41:32 jonblower
126 * Closed PrintStreams properly
127 *
128 * Revision 1.2 2005/10/16 22:05:28 jonblower
129 * Improved handling of output streams (mapped CStyxFiles to PrintStreams)
130 *
131 * Revision 1.1 2005/10/14 17:57:18 jonblower
132 * Added SGSRun and associated shell scripts
133 *
134 */
135 public class SGSRun extends SGSInstanceClientChangeAdapter implements StyxConnectionListener
136 {
137 /***
138 * The exit code that signals an internal error in the SGS mechanism, rather
139 * than an error code from the executable that is running on the SGS server
140 */
141 public static final int INTERNAL_SGS_ERROR = 20000;
142
143 private static final Logger log = Logger.getLogger(SGSRun.class);
144
145 private static final String OUTPUT_ALL_REFS = "sgs-allrefs";
146 private static final String HELP = "sgs-help";
147 private static final String VERBOSE_HELP = "sgs-verbose-help";
148 private static final String LIFETIME = "sgs-lifetime";
149 private static final String DEBUG = "sgs-debug";
150 private static final StringParser POSITIVE_FLOAT_PARSER =
151 PositiveFloatStringParser.getParser();
152
153 private String hostname;
154 private int port;
155 private String serviceName;
156 private SGSServerClient serverClient;
157 private SGSClient sgsClient;
158 private SGSInstanceClient instanceClient;
159 private int exitCode;
160 private boolean allDataDownloaded;
161
162 private JSAPResult result;
163
164 private boolean debug;
165
166 private SGSConfig config;
167
168 private Hashtable
169
170
171 /***
172 * Creates a new SGSRun object
173 * @param hostname The name (or IP address) of the SGS server
174 * @param port The port of the SGS server
175 * @param serviceName The name of the SGS to invoke
176 */
177 public SGSRun(String hostname, int port, String serviceName)
178 {
179 this.hostname = hostname;
180 this.port = port;
181 this.serviceName = serviceName;
182 this.debug = false;
183 this.allDataDownloaded = false;
184 this.exitCode = Integer.MIN_VALUE;
185 this.filesToUpload = new Hashtable();
186 }
187
188 /***
189 * Connects to the SGS server and downloads the configuration of this
190 * SGS.
191 * @throws StyxException if there was an error connecting to the server
192 * or getting the configuration of the SGS
193 * @throws UnknownHostException (from SGSServerClient.getSGSClient())
194 */
195 public void connect() throws StyxException, UnknownHostException
196 {
197
198 this.serverClient = SGSServerClient.getServerClient(this.hostname, this.port);
199 log.debug("Connected to SGS server at " + this.hostname + ":" + this.port);
200
201
202 this.sgsClient = serverClient.getSGSClient(this.serviceName);
203 log.debug("Got handle to the " + this.serviceName + " Styx Grid Service");
204
205
206 this.config = this.sgsClient.getConfig();
207 log.debug("Got configuration information for the " + this.serviceName +
208 " Styx Grid Service");
209 }
210
211 /***
212 * Gets the JSAP object from the config object and adds the extra parameters
213 * we need to parse the command line
214 */
215 private JSAP getJSAP() throws StyxException
216 {
217
218 JSAP jsap = this.config.getParamParser();
219
220 try
221 {
222
223 jsap.registerParameter(new Switch(HELP, JSAP.NO_SHORTFLAG, HELP,
224 "Set this switch to print out a short help message"));
225
226 jsap.registerParameter(new Switch(VERBOSE_HELP, JSAP.NO_SHORTFLAG, VERBOSE_HELP,
227 "Set this switch to print out a long help message"));
228
229 jsap.registerParameter(new Switch(DEBUG, JSAP.NO_SHORTFLAG, DEBUG,
230 "Set this switch in order to enable printing of debug messages"));
231
232
233 jsap.registerParameter(new Switch(OUTPUT_ALL_REFS, JSAP.NO_SHORTFLAG, OUTPUT_ALL_REFS,
234 "Set this switch in order to get URLs to all output files rather than actual files"));
235
236
237 jsap.registerParameter(new FlaggedOption(LIFETIME, POSITIVE_FLOAT_PARSER,
238 "60", false, JSAP.NO_SHORTFLAG, LIFETIME,
239 "The lifetime of the SGS in minutes"));
240
241
242
243 Vector inputs = this.config.getInputs();
244 for (int i = 0; i < inputs.size(); i++)
245 {
246 SGSInput input = (SGSInput)inputs.get(i);
247 if (input.getType() == SGSInput.FILE)
248 {
249 jsap.registerParameter(new FlaggedOption("sgs-ref-" + input.getName(), JSAP.STRING_PARSER,
250 null, false, JSAP.NO_SHORTFLAG, "sgs-ref-" + input.getName(),
251 "If set, will cause the input file " + input.getName() +
252 " to be uploaded from the given URL"));
253 }
254 else if (input.getType() == SGSInput.STREAM)
255 {
256 jsap.registerParameter(new FlaggedOption("sgs-ref-" + input.getName(), JSAP.STRING_PARSER,
257 null, false, JSAP.NO_SHORTFLAG, "sgs-ref-" + input.getName(),
258 "If set, will cause the data at the given URL to be redirected" +
259 " to the " + input.getName() + " stream of the service"));
260 }
261 }
262
263
264
265
266 Vector outputs = this.config.getOutputs();
267 for (int i = 0; i < outputs.size(); i++)
268 {
269 SGSOutput output = (SGSOutput)outputs.get(i);
270 if (output.getType() == SGSOutput.FILE ||
271 output.getType() == SGSOutput.STREAM)
272 {
273
274 jsap.registerParameter(new Switch("sgs-ref-" + output.getName(),
275 JSAP.NO_SHORTFLAG, "sgs-ref-" + output.getName(),
276 "If set, will output a URL to " + output.getName() +
277 " instead of the actual data"));
278 }
279 }
280 }
281 catch (JSAPException jsape)
282 {
283 throw new StyxException(jsape.getMessage());
284 }
285
286 return jsap;
287 }
288
289 /***
290 * Checks the command-line arguments: makes sure they can be parsed and checks
291 * for the existence of all input files
292 * @throws StyxException if the arguments are not valid
293 */
294 public void checkArguments(String[] args) throws StyxException
295 {
296 JSAP parser = this.getJSAP();
297 this.result = parser.parse(args);
298 log.debug("Parsed command-line arguments, success = " + this.result.success());
299
300
301 if (this.result.getBoolean(VERBOSE_HELP))
302 {
303 System.out.println(this.config.getDescription());
304 System.out.println("");
305 System.out.println("Usage: " + this.sgsClient.getName() + " " +
306 parser.getUsage());
307 System.out.println("");
308 System.out.println(parser.getHelp());
309 System.exit(0);
310 }
311 else if (this.result.getBoolean(HELP))
312 {
313 System.out.println(this.config.getDescription());
314 System.out.println("");
315 System.out.println("Usage: " + this.sgsClient.getName() + " " +
316 parser.getUsage());
317 System.exit(0);
318 }
319
320 if (this.result.success())
321 {
322
323 this.debug = this.result.getBoolean(DEBUG);
324
325 Vector inputs = this.config.getInputs();
326 for (int i = 0; i < inputs.size(); i++)
327 {
328 SGSInput input = (SGSInput)inputs.get(i);
329
330 String url = this.result.getString("sgs-ref-" + input.getName());
331 if (url != null && !url.trim().equals(""))
332 {
333
334 if (this.debug)
335 {
336 System.out.println("Set URL for " + input.getName() +
337 ": " + url);
338 }
339 if (url.startsWith("readfrom:"))
340 {
341 this.filesToUpload.put(input, url);
342 }
343 else
344 {
345
346 this.filesToUpload.put(input, "readfrom:" + url);
347 }
348 }
349 else
350 {
351
352 if (input.getType() == SGSInput.FILE)
353 {
354 File f = new File(input.getName());
355 if (f.exists())
356 {
357 this.filesToUpload.put(input, f);
358 }
359 else
360 {
361
362 throw new StyxException("File " + input.getName() +
363 " does not exist");
364 }
365 }
366 }
367 }
368 log.debug("Scheduled files for upload");
369 }
370 else
371 {
372
373 System.err.println("Usage: " + this.sgsClient.getName() + " " +
374 parser.getUsage());
375 Iterator errIt = this.result.getErrorMessageIterator();
376 String errMsg = "Error occurred parsing command line: ";
377 errMsg += errIt.hasNext() ? (String)errIt.next() : "no details";
378 throw new StyxException(errMsg);
379 }
380 }
381
382 /***
383 * Creates a new service instance
384 */
385 public void createNewServiceInstance() throws StyxException
386 {
387 String instanceUrl = this.sgsClient.createNewInstance();
388 log.debug("Created new service instance at " + instanceUrl);
389
390
391
392 this.instanceClient = new SGSInstanceClient(instanceUrl);
393 log.debug("Created new SGSInstanceClient object");
394 this.instanceClient.setLifetime(this.result.getFloat(LIFETIME));
395 log.debug("Set lifetime to " + this.result.getFloat(LIFETIME) + " minutes");
396 this.instanceClient.addChangeListener(this);
397
398
399 if (this.serverClient.getConnection() != this.instanceClient.getConnection())
400 {
401 log.debug("Closing connection to SGS server");
402 this.serverClient.getConnection().close();
403 log.debug("Connection to SGS server closed");
404 }
405 this.instanceClient.getConnection().addListener(this);
406 }
407
408 /***
409 * Sets the values of all the parameters
410 */
411 public void setParameters() throws StyxException, FileNotFoundException
412 {
413
414 Vector params = this.config.getParams();
415 log.debug("Got " + params.size() + " parameters");
416 for (Iterator it = params.iterator(); it.hasNext(); )
417 {
418 SGSParam param = (SGSParam)it.next();
419 log.debug("Got parameter called " + param.getName());
420 if (param.getType() == SGSParam.OUTPUT_FILE)
421 {
422
423 }
424 else
425 {
426
427 log.debug("Setting value for " + param.getName());
428 if (param.getParameter() instanceof Switch)
429 {
430 String val = this.result.getBoolean(param.getName()) ? "true" : "false";
431 log.debug("val = " + val);
432 this.instanceClient.setParameterValue(param, val);
433 }
434 else
435 {
436 this.instanceClient.setParameterValue(param,
437 this.result.getStringArray(param.getName()));
438 }
439 }
440 }
441 log.debug("Set service parameters");
442 }
443
444 /***
445 * Sets the input sources for the Styx Grid Service
446 */
447 public void setInputSources() throws StyxException
448 {
449
450 for (Enumeration en = this.filesToUpload.keys(); en.hasMoreElements(); )
451 {
452 SGSInput inputFile = (SGSInput)en.nextElement();
453 Object dataSrc = this.filesToUpload.get(inputFile);
454 try
455 {
456 if (dataSrc instanceof String)
457 {
458 this.instanceClient.setInputSource(inputFile, (String)dataSrc);
459 }
460 else
461 {
462
463 this.instanceClient.setInputSource(inputFile, (File)dataSrc);
464 }
465 }
466 catch(FileNotFoundException fnfe)
467 {
468
469
470 throw new StyxException("Internal error: " + fnfe.getMessage());
471 }
472 }
473
474 this.instanceClient.uploadInputFiles();
475 log.debug("All input sources set");
476 }
477
478 /***
479 * Starts the service and begins reading from the output streams
480 * @throws StyxException if there was an error starting the service
481 */
482 public void start() throws StyxException
483 {
484
485 this.instanceClient.startService();
486 log.debug("Service started");
487 }
488
489 /***
490 * Starts reading from the output files. Note that we have
491 * already set the destinations for the outputs that are set via a parameter
492 * in the setParameters() method
493 */
494 public void readOutputFiles() throws StyxException, FileNotFoundException
495 {
496
497
498 boolean allRefs = this.result.getBoolean(OUTPUT_ALL_REFS);
499 Vector outputFiles = this.config.getOutputs();
500 boolean downloading = false;
501 for (int i = 0; i < outputFiles.size(); i++)
502 {
503 SGSOutput output = (SGSOutput)outputFiles.get(i);
504 if (output.getType() == SGSOutput.FILE_FROM_PARAM)
505 {
506
507
508 String filename = this.result.getStringArray(output.getName())[0].trim();
509 PrintStream prtStr = this.getPrintStream(filename);
510 if (allRefs || filename.endsWith(".sgsref"))
511 {
512
513 prtStr.print("readfrom:" +
514 this.instanceClient.getOutputFileURL(output.getName()));
515 prtStr.close();
516 }
517 else
518 {
519
520
521 this.instanceClient.redirectOutput(output.getName(), prtStr);
522 log.debug("Started reading from " + output.getName());
523 downloading = true;
524 }
525 }
526 else
527 {
528
529 PrintStream prtStr = this.getPrintStream(output.getName());
530 if (allRefs || this.result.getBoolean("sgs-ref-" + output.getName()))
531 {
532 prtStr.print("readfrom:" + this.instanceClient.getOutputFileURL(output.getName()));
533 if (prtStr != System.out && prtStr != System.err)
534 {
535 prtStr.close();
536 }
537 }
538 else
539 {
540 this.instanceClient.redirectOutput(output.getName(), prtStr);
541 log.debug("Started reading from " + output.getName());
542 downloading = true;
543 }
544 }
545 }
546 if (!downloading)
547 {
548 this.allOutputDataDownloaded();
549 }
550 }
551
552 /***
553 * @return a PrintStream for the given output file name. If a file with the
554 * given name already exists, this truncates the file to zero length.
555 * @throws FileNotFoundException if the file exists but is a directory.
556 */
557 private PrintStream getPrintStream(String filename) throws FileNotFoundException
558 {
559 if (filename.equals("stdout"))
560 {
561 return System.out;
562 }
563 else if (filename.equals("stderr"))
564 {
565 return System.err;
566 }
567 else
568 {
569
570
571 return new PrintStream(new FileOutputStream(filename));
572 }
573 }
574
575 /***
576 * Called when the given service data element changes
577 */
578 public void gotServiceDataValue(String sdName, String newData)
579 {
580 if (this.debug)
581 {
582 System.out.println(sdName + " = " + newData);
583 }
584 }
585
586 /***
587 * Called when the exit code from the service is received: this signals that
588 * the remote executable has completed.
589 */
590 public void gotExitCode(int exitCode)
591 {
592 log.debug("Got exit code: " + exitCode);
593 this.exitCode = exitCode;
594 if (this.allDataDownloaded)
595 {
596 this.instanceClient.close();
597 }
598 }
599
600 /***
601 * Called when all the output data have been downloaded
602 */
603 public void allOutputDataDownloaded()
604 {
605 log.debug("All data downloaded");
606 this.allDataDownloaded = true;
607 if (this.exitCode != Integer.MIN_VALUE)
608 {
609 this.instanceClient.close();
610 }
611 }
612
613 /***
614 * Called when the connection to the SGS instance has been closed
615 */
616 public void connectionClosed(StyxConnection conn)
617 {
618 log.debug("Exiting with code " + this.exitCode);
619 System.exit(this.exitCode);
620 }
621
622 /***
623 * Called when the relevant handshaking has been performed and the connection
624 * is ready for Styx messages to be sent. Required by StyxConnectionListener
625 * interface.
626 */
627 public void connectionReady(StyxConnection conn) {}
628
629 /***
630 * Called when an error has occurred when connecting. Required by
631 * StyxConnectionListener interface.
632 * @param message String describing the problem
633 */
634 public void connectionError(StyxConnection conn, String message) {}
635
636 public static void main(String[] args)
637 {
638 if (args.length < 3)
639 {
640 System.err.println("Usage: SGSRun <hostname> <port> <servicename> [args]");
641 System.exit(INTERNAL_SGS_ERROR);
642 }
643
644
645 System.setProperty("java.protocol.handler.pkgs", "uk.ac.rdg.resc.jstyx.client.protocol");
646
647 SGSRun runner = null;
648 try
649 {
650 int port = Integer.parseInt(args[1]);
651
652
653 String[] sgsArgs = new String[args.length - 3];
654 System.arraycopy(args, 3, sgsArgs, 0, sgsArgs.length);
655
656
657 runner = new SGSRun(args[0], port, args[2]);
658
659
660
661 runner.connect();
662
663
664 runner.checkArguments(sgsArgs);
665
666
667 runner.createNewServiceInstance();
668
669
670 runner.setParameters();
671
672
673 runner.setInputSources();
674
675
676 runner.start();
677
678
679 runner.readOutputFiles();
680
681
682
683 }
684 catch(NumberFormatException nfe)
685 {
686 System.err.println("Invalid port number");
687
688 System.exit(INTERNAL_SGS_ERROR);
689 }
690 catch(Exception e)
691 {
692 if (log.isDebugEnabled())
693 {
694 e.printStackTrace();
695 }
696 System.err.println("Error running Styx Grid Service: " + e.getMessage());
697
698 if (runner != null)
699 {
700 runner.exitCode = INTERNAL_SGS_ERROR;
701
702
703 if (runner.serverClient != null)
704 {
705 runner.serverClient.getConnection().close();
706 }
707 if (runner.instanceClient != null)
708 {
709 runner.instanceClient.getConnection().close();
710 }
711 }
712 }
713 }
714
715 }
716
717 /***
718 * This class parses strings into positive floating-point numbers, throwing
719 * a ParseException if the string could not be parsed. Use getParser() to get
720 * an instance of this class.
721 */
722 class PositiveFloatStringParser extends StringParser
723 {
724 private static PositiveFloatStringParser parser = null;
725 private StringParser floatStrParser;
726
727 private PositiveFloatStringParser()
728 {
729 this.floatStrParser = JSAP.FLOAT_PARSER;
730 }
731
732 public static PositiveFloatStringParser getParser()
733 {
734 if (parser == null)
735 {
736 parser = new PositiveFloatStringParser();
737 }
738 return parser;
739 }
740
741 /***
742 * Parses the given string into a positive Float
743 */
744 public Object parse(String arg) throws ParseException
745 {
746 Float f = (Float)this.floatStrParser.parse(arg);
747 if (f.floatValue() < 0.0f)
748 {
749 throw new ParseException(arg + " must be a positive number");
750 }
751 return f;
752 }
753 }