#!/bin/sh
# startEpicsIoc.sh
#
# dispatcher for startEpicsIoc.sh
# - checks for matching hostnames and
#   calls $myCalleeScript starting IOC in background using procServ depending on HOSTNAME content
#
# created: 13-01-2010
# author: p.zumbruch, GSI
# modified: 08-02-2010

#debug - uncomment to debug
# set -xv

#specific predefines function
specificPredefines()
{
	# more specific variables to be (un)set before running

	myHostname=$(hostname)
	myBaseHome=/home/hadaq
	myStartupProcedures=${myBaseHome}/EPICS/startupProcedures
	myCalleeScript=${myStartupProcedures}/startBackgroundEpicsIoc.sh

	myProcServTelnetPort=4813
	unset myProcServTelnetLogPort #default: telnet port + 1
	unset myProcServIocName # set by callee
	unset myWorkingDir #
	unset commandExecutable
	unset commandArguments
	unset myLoggingDir
	unset myIocBootDir
	myProcServExecutable=$(which procServ)
	myProcServDefaultExecutable=${myStartupProcedures}/procServ/procServ
	myProcServExecutable=${myProcServExecutable:-${myProcServDefaultExecutable}}
	myLdLibPaths=${myStartupProcedures}/procServ/lib
	myList='etrax046 etrax047 etrax048'
	unset multipleIocs
}

#predefines function
predefines()
{
	# general variables to be (un)set before running

	scriptname=startEpicsIoc.sh
	myExitIfHelperScript=$(dirname $0)/exitIfHelpers.sh

	unset myIocListFile
	myIocListFileSeparator='|'
	unset calleeArguments
	unset myProcServFileLogging

	myVerbosity=0;
	myVerbosityCallee=0;
	unset myDebug
	unset myDebugCallee
	unset myEcho;
	unset myEchoCallee;
    unset myHelp
	unset myQuiet;
	unset myQuietCallee;

	myLevel=0
	myListLength=0;
}

#unset function
unsets()
{
	set +xv
	# cleaning up, i.e. unsetting all variables used
	unset scriptname
	unset myHostname
	unset myCalleeScript
	unset myList
	unset myIocListFile
	unset myWorkingDir
	unset myIocBootDir
	unset commandExecutable
	unset commandArguments
	unset calleeArguments
	unset myProcServFileLogging
	unset myLoggingDir
	unset myProcServTelnetPort
	unset myProcServTelnetLogPort
	unset myProcServExecutable
	unset myProcServDefaultExecutable
	unset myLdLibPaths
	unset myBaseHome
	unset myEcho
	unset myEchoCallee
	unset myDebug
	unset myDebugCallee
	unset myVerbosity
	unset myVerbosityCallee
	unset myExitIfHelperScript
	unset oldIFS
        unset myHelp
	unset myIocListFileSeparator
	unset myListLength
	unset myProcServIocName

	unset myHostnameOpt;              unset myHostnameOptArg
	unset myCalleeScriptOpt;          unset myCalleeScriptOptArg
	unset myExitIfHelperScriptOpt;    unset myExitIfHelperScriptOptArg
	unset myListOpt;                  unset myListOptArg
	unset multipleIocsOpt;
	unset myIocListFileOpt;           unset myIocListFileOptArg
	unset myIocListFileSeparatorOpt;  unset myIocListFileSeparatorOptArg
	unset myWorkingDirOpt;            unset myWorkingDirOptArg
	unset myIocBootDirOpt;            unset myIocBootDirOptArg
	unset commandExecutableOpt;       unset commandExecutableOptArg
	unset commandArgumentsOpt;        unset commandArgumentsOptArg
	unset calleeArgumentsOpt;         unset calleeArgumentsOptArg
	unset myProcServFileLoggingOpt;
	unset myLoggingDirOpt;            unset myLoggingDirOptArg
	unset myProcServTelnetPortOpt;    unset myProcServTelnetPortOptArg
	unset myProcServTelnetLogPortOpt; unset myProcServTelnetLogPortOptArg
	unset myProcServExecutableOpt;    unset myProcServExecutableOptArg
	unset myLdLibPathsOpt;            unset myLdLibPathsOptArg
	unset myBaseHomeOpt;              unset myBaseHomeOptArg
	unset myEchoOpt;
	unset myEchoCalleeOpt;
	unset myDebugOpt;
	unset myDebugCalleeOpt;
	unset myVerbosityOpt;
	unset myVerbosityCalleeOpt;
	unset myHelpOpt;
	unset myProcServIocNameOpt;       unset myProcServIocNameOptArg
	return 0;
}

#dump function
dump()
{
	# print out all necessary information on variables used and set
	local function=dump
	[ 3 -le $myVerbosity ] && myLevel=$(($myLevel+1)) && echo [${myLevel}] entering function \"$function $*\"

	echo -e " settings are:\n"
	local mv=$myVerbosity;
	local mvc=$myVerbosityCallee;
	echo -e "                \"scriptname \": \t" ${scriptname}
	echo -e "                \"myHostname \": \t" ${myHostname}
	echo -e "      \"myExitIfHelperScript \": \t" ${myExitIfHelperScript}
	echo -e "            \"myCalleeScript \": \t" ${myCalleeScript}
	echo -e "                    \"myList \": \t" ${myList}
	echo -e "             \"myIocListFile \": \t" ${myIocListFile}
	echo -e "    \"myIocListFileSeparator \": \t" ${myIocListFileSeparator}
	echo -e "              \"myWorkingDir \": \t" ${myWorkingDir}
	echo -e "              \"myIocBootDir \": \t" ${myIocBootDir}
	echo -e "         \"commandExecutable \": \t" ${commandExecutable}
	echo -e "          \"commandArguments \": \t" ${commandArguments}
	echo -e "           \"calleeArguments \": \t" ${calleeArguments}
	echo -e "         \"myProcServIocName \": \t" ${myProcServIocName}
	echo -e "     \"myProcServFileLogging \": \t" "[$([ -z ${myProcServFileLogging} ] && echo OFF || echo ON)]"
	echo -e "              \"multipleIocs \": \t" "[$([ -z ${multipleIocs} ] && echo OFF || echo ON)]"
	echo -e "              \"myLoggingDir \": \t" ${myLoggingDir}
	echo -e "      \"myProcServTelnetPort \": \t" ${myProcServTelnetPort}
	echo -e "   \"myProcServTelnetLogPort \": \t" ${myProcServTelnetLogPort}
	echo -e "      \"myProcServExecutable \": \t" ${myProcServExecutable}
	echo -e "              \"myLdLibPaths \": \t" ${myLdLibPaths}
	echo -e "                \"myBaseHome \": \t" ${myBaseHome}
	echo -e "                    \"myEcho \": \t" "[$([ -z ${myEcho} ]		&& echo OFF || echo ON)]"
	echo -e "              \"myEchoCallee \": \t" "[$([ -z ${myEchoCallee} ]	&& echo OFF || echo ON)]"
	echo -e "                   \"myDebug \": \t" "[$([ -z ${myDebug} ]		&& echo OFF || echo ON)]"
	echo -e "             \"myDebugCallee \": \t" "[$([ -z ${myDebugCallee} ]	&& echo OFF || echo ON)]"
	echo -e "               \"myVerbosity \": \t" "[$([ 0 -eq ${mv} ]		&& echo OFF || echo ON)$([ 1 -lt ${mv} ] && echo \($mv\))]"
	echo -e "         \"myVerbosityCallee \": \t" "[$([ 0 -eq ${mvc} ]		&& echo OFF || echo ON)$([ 1 -lt ${mvc} ] && echo \($mvc\))]"

	[ 3 -le $myVerbosity ] && echo [${myLevel}] " leaving" function \"$function\" && myLevel=$(($myLevel-1))
}

#usage function
usage()
{
	# How to use the script

	echo
	echo    "$scriptname"
	echo    "usage: $scriptname [options]"
	echo
	echo    " caller for script calling EPICS IOCs with procServ in \"Background\","
	echo    "  checks for matching hostnames with a given list of hostname,"
	echo    "  or a file with a lookuptable to be evaluated"
	echo    "  and calls the start script with the right argument"
	echo
	echo    "options:"
	echo
	echo    "   Caller: "
	echo -e    "   -"${myHostnameOpt}              "hostname     : hostname"
	echo -e    "                       [\`${myHostname}']"
	echo -e    "   -"${myCalleeScriptOpt}          "scriptname   : background starting script's name (absolute path) "
	echo -e    "                       [\`${myCalleeScript}']"
	echo -e    "   -"${myExitIfHelperScriptOpt}    "scriptname   : helper script internally used by \`$scriptname' for checks"
	echo -e    " 			    defaults to be searched in the same directory as \'$scriptname'"
	echo -e    "                       [\`${myExitIfHelperScript}']"
	echo -e    "   -"${myListOpt}                  "list         : quoted space separated list of hostnames, where an IOC should be started"
	echo -e    "                       [\`\"$myList\"']"
	echo -e    "   -"${multipleIocsOpt}            "             : allow several IOCs running [needs differing telnet ports to monitor/control]"
	echo -e    "                       [\`\"$multipleIocs\"']"
	echo -e    "   -"${myIocListFileOpt}           "file         : file to retrieve hostnames, executables, dirs, arguments from, "
	echo -e    "                       overwrites all other relevant/default settings (absolute path)"
	echo -e    "                       [\`$myIocListFile']"
	echo -e    "   -"${myIocListFileSeparatorOpt}  "char         : character separating the entries in the ioc list file"
	echo -e    "                       [\`$myIocListFileSeparator']"
	echo -e
	echo -e    "   Callee: "
	echo -e    "   -"${myWorkingDirOpt}            "workingDir   : place where log (default) and pid files may be written (absolute path)"
	echo -e    "                       [\`$myWorkingDir']"
	echo -e    "   -"${myIocBootDirOpt}            "iocBootDir   : place where to change to IOC's boot directory,"
	echo -e    "                       if not absolute path, taken relatively to \`workingDir'"
	echo -e    "                       [\`$myIocBootDir']"
	echo -e    "   -"${commandExecutableOpt}       "commandExe   : executable to be executed by callee,"
	echo -e    "                       if not absolute path, taken relatively to \`workingDir'"
	echo -e    "                       [\`$commandExecutable']"
	echo -e    "   -"${commandArgumentsOpt}        "commandArgs  : [quoted list] of argument[s] for executable to be executed by callee,"
	echo -e    "                       [\`\"$commandArguments\"']"
	echo -e    "   -"${calleeArgumentsOpt}         "commandArgs  : [quoted list] of [additional] argument[s] for callee,"
	echo -e    "                       [\`\"$calleeArguments\"']"
	echo -e    "   -"${myProcServIocNameOpt}       "name         : name to assign to the process (ioc) the callee starts"
	echo -e    "                       [$([ -z ${myProcServIocName} ] && echo automatically set by callee || echo \`\"$myProcServIocName\"\')]"
	echo -e    "   -"${myProcServFileLoggingOpt}   "             : activate file logging for procServ [$([ -z "${myProcServFileLogging}" ] && echo -e OFF || echo -e ON)]"
	echo -e    "   -"${myLoggingDirOpt}            "loggingDir   : file logging directory,"
	echo -e    "                       if not absolute path, taken relatively to \`workingDir' [\`$myWorkingDir']"
	echo -e    "                       [\`$myLoggingDir']"
	echo -e    "   -"${myProcServTelnetPortOpt}    "port         : set telnet port for active procServ access, (NOTE: readonly port will be port+1)"
	echo -e    "                       [\`$myProcServTelnetPort']"
	echo -e    "   -"${myProcServTelnetLogPortOpt} "logport      : set log readonly telnet port for active procServ access (NOTE: default log port will be port+1)"
	echo -e    "                       [\`$myProcServTelnetLogPort']"
	echo -e    "   -"${myProcServExecutableOpt}    "procServExe  : procServ executable"
	echo -e    "                       if not absolute path, taken relatively to \`baseHome' [\`$myBaseHome']"
	echo -e    "                       [\`$myProcServExecutable']"
	echo -e    "   -"${myLdLibPathsOpt}            "             : [list of colon ':' separated] shared library paths to be added"
	echo -e    "                       to LD_LIBRARY_PATH (may be called repeatedly)"
	echo -e    "                       [\`$myLdLibPaths']"
	echo -e
	echo -e    "   General: "
	echo -e    "   -"${myBaseHomeOpt}              "baseHome     : home directory of hadaq"
	echo -e    "                       [\`$myBaseHome']"
	echo -e -n "   -"${myEchoOpt}                  "             : no caller action, just echos command instead of executing the caller"
	echo -e    " [$([ -z ${myEcho} ] && echo OFF || echo ON)]"
	echo -e -n "   -"${myEchoCalleeOpt}            "             : no callee action, just echos command instead of executing the callee"
	echo -e    " [$([ -z ${myEchoCallee} ] && echo OFF || echo ON)]"
	echo -e -n "   -"${myDebugOpt}                 "             : debug mode of caller, sets set -xv"
	echo -e 	" [$([ -z ${myDebug} ] && echo OFF || echo ON)]"
	echo -e -n "   -"${myDebugCalleeOpt}           "             : debug mode of callee, sets set -xv"
	echo -e 	" [$([ -z ${myDebugCallee} ] && echo OFF || echo ON)]"
	echo -e -n "   -"${myVerbosityOpt}             "             : verbosity of caller, can be called repeatedly "
	echo -e 	" [$([ 0 -eq ${myVerbosity} ] && echo OFF || echo ON)$([ 1 -lt ${myVerbosity} ] && echo \($myVerbosity\))]"
	echo -e -n "   -"${myVerbosityCalleeOpt}       "             : verbosity of callee, can be called repeatedly "
	echo -e 	" [$([ 0 -eq ${myVerbosityCallee} ] && echo OFF || echo ON)$([ 1 -lt ${myVerbosityCallee} ] && echo \($myVerbosityCallee\))]"
	echo -e -n "   -"${myQuietOpt}                 "             : sets verbosity of caller off"
	echo 	   " [$([ -z ${myQuiet} ] && echo OFF || echo ON)]"
	echo -e -n "   -"${myQuietCalleeOpt}           "             : sets verbosity of callee off "
	echo -e 	" [$([ -z ${myQuietCallee} ] && echo OFF || echo ON)]"
	echo -e    "   -"${myHelpOpt}                  "             : this help"
	echo
}

#eval options function
evalOptions()
{
	#evalOptions $@
	#evaluates the options of the command line

	#options definitions
	             myHostnameOpt="H";              myHostnameOptArg=':'
	         myCalleeScriptOpt="s";          myCalleeScriptOptArg=':'
	   myExitIfHelperScriptOpt="e";    myExitIfHelperScriptOptArg=':'
	                 myListOpt="l";                  myListOptArg=':'
	           multipleIocsOpt="m";
	          myIocListFileOpt="f";           myIocListFileOptArg=':'
	 myIocListFileSeparatorOpt="S";  myIocListFileSeparatorOptArg=':'
	           myWorkingDirOpt="w";            myWorkingDirOptArg=':'
	           myIocBootDirOpt="i";            myIocBootDirOptArg=':'
	      commandExecutableOpt="c";       commandExecutableOptArg=':'
	       commandArgumentsOpt="a";        commandArgumentsOptArg=':'
	        calleeArgumentsOpt="A";         calleeArgumentsOptArg=':'
          myProcServFileLoggingOpt="g";
	           myLoggingDirOpt="G";            myLoggingDirOptArg=':'
	   myProcServTelnetPortOpt="p";    myProcServTelnetPortOptArg=':'
	myProcServTelnetLogPortOpt="t"; myProcServTelnetLogPortOptArg=':'
	   myProcServExecutableOpt="P";    myProcServExecutableOptArg=':'
	           myLdLibPathsOpt="L";            myLdLibPathsOptArg=':'
	             myBaseHomeOpt="b";              myBaseHomeOptArg=':'
	      myProcServIocNameOpt="I";       myProcServIocNameOptArg=':'
	                 myEchoOpt="n";
	           myEchoCalleeOpt="N";
	                myDebugOpt="d";
	          myDebugCalleeOpt="D";
	            myVerbosityOpt="v";
	      myVerbosityCalleeOpt="V";
	                myQuietOpt="q";
	          myQuietCalleeOpt="Q";
	                 myHelpOpt="h";

	local opts="myHostname myCalleeScript myExitIfHelperScript myList multipleIocs myIocListFile myWorkingDir myIocBootDir commandExecutable commandArguments calleeArguments myProcServFileLogging myLoggingDir myProcServTelnetPort myProcServTelnetLogPort myProcServExecutable myLdLibPaths myBaseHome myEcho myEchoCallee myDebug myDebugCallee myVerbosity myVerbosityCallee myQuiet myQuietCallee myHelp myIocListFileSeparator myProcServIocName"

	for item in $(echo $opts | sed -e "s/ /$IFS/g")
	do
		options="${options}\${${item}Opt}\${${item}OptArg}"
  	done
    	options=$(eval echo $options)

	while getopts $options arg
	do
		case $arg in
			${myHostnameOpt})		myHostname=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${myCalleeScriptOpt})		myCalleeScript=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${myListOpt})			myList=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${multipleIocsOpt})		multipleIocs=yes
				shift;;
			${myIocListFileOpt})		myIocListFile=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${myIocListFileSeparatorOpt})	myIocListFileSeparator=$(echo $OPTARG | sed -e 's/"//g' | sed 's/(.).*/\1/g')
				shift 2;;
			${myWorkingDirOpt})		myWorkingDir=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${myIocBootDirOpt})		myIocBootDir=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${commandExecutableOpt})	commandExecutable=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${commandArgumentsOpt})		commandArguments=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${calleeArgumentsOpt})		calleeArguments=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${myProcServFileLoggingOpt})	myProcServFileLogging=yes
				shift;;
			${myProcServIocNameOpt})	myProcServIocName=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${myLoggingDirOpt})		myLoggingDir=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${myProcServTelnetPortOpt})	myProcServTelnetPort=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${myProcServTelnetLogPortOpt})	myProcServTelnetLogPort=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${myProcServExecutableOpt})	myProcServExecutable=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${myLdLibPathsOpt} )		[ -d $(echo $OPTARG | sed -e 's/"//g') ] &&
							myLdLibPaths=${myLdLibPaths:+${myLdLibPaths}\:}$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${myBaseHomeOpt})		myBaseHome=$(echo $OPTARG | sed -e 's/"//g')
				shift 2;;
			${myExitIfHelperScriptOpt})	myExitIfHelperScript=$(echo $OPTARG | sed -e 's/"//g')
							[ ! -z "${myExitIfHelperScript##/*}" ] && myExitIfHelperScript="$(cd $(dirname ${0}) 2>/dev/null && pwd)/${myExitIfHelperScript}"
				shift 2;;
			${myEchoOpt})			myEcho=echo
				shift;;
			${myEchoCalleeOpt})		myEchoCallee=echo
				shift;;
			${myDebugOpt})			myDebug=yes
				shift;;
			${myDebugCalleeOpt})		myDebugCallee=yes
				shift;;
			${myVerbosityOpt})		myVerbosity=$(($myVerbosity + 1))
				shift;;
			${myVerbosityCalleeOpt})	myVerbosityCallee=$(($myVerbosityCallee + 1))
				shift;;
			${myQuietOpt})			myQuiet=yes
							myVerbosity=0
				shift;;
			${myQuietCalleeOpt})		myQuietCallee=yes
							myVerbosityCallee=0
				shift;;
			${myHelpOpt} )			myHelp=yes
				shift;;
		esac
	done
	return 0;
}

#checks function
checks()
{
	# checks and twiddling run
	# performs checks on all settings
	# depending on the run number (allowed: {1,2,}
	# and complete path and pathnames
	# different stages are checked

	local function=checks
	[ 3 -le $myVerbosity ] && myLevel=$(($myLevel+1)) && echo [${myLevel}] entering function \"$function $*\"

	if [ 1 -eq "$1" ]
	then
		#checks first round
		[ 3 -le $myVerbosity ] && echo [${myLevel}] \"... checks first round\"

		# Checks for non empty
		# - mandatory callee script
		if [ -z "${myCalleeScript}" ]
		then
			echo "$scriptname: callee script name not set or empty ... exiting" 1>&2
			unsets
			exit 1;
		fi
		# - mandatory hostname
		if [ -z "${myHostname}" ]
		then
			echo "$scriptname: hostname not available, set or empty ... exiting" 1>&2
			unsets
			exit 1;
		fi


		# Check is file / directory and complete path
		# - optional IocListFile
		if [ ! -z ${myIocListFile} ]
		then
			[ 3 -le $myVerbosity ] && echo [${myLevel}] "\"   ... IOC list file:\""
			isAbsolutePath ${myIocListFile} 	       || myIocListFile=$(pwd)/${myIocListFile}
			exitIfNotAFile ${myIocListFile} "IOC list file"
			myIocListFile="$(cd $(dirname ${myIocListFile}) 2>/dev/null && pwd)/$(basename ${myIocListFile})"
		fi

		# Checks for mandatory absolute paths, if not exit
		exitIfNotAbsolutePath $myCalleeScript  "callee script"
		[ ! -z ${myCalleeScript}  ] &&
			exitIfNotAbsolutePath $myCalleeScript "callee script"
		# Executable permission
		# - mandatory callee script

		exitIfNotExecutable ${myCalleeScript} "caller script" &&
			myCalleeScript="$(cd $(dirname ${myCalleeScript}) 2>/dev/null && pwd)/$(basename ${myCalleeScript})"
	elif [ 2 -eq "$1" ]
	then
		#checks second round
		[ 3 -le $myVerbosity ] && echo [${myLevel}] \"... checks second round\"

		# Checks for mandatory absolute paths, if not exit

		[ 3 -le $myVerbosity ] && echo [${myLevel}] "\"... checks for mandatory absolute paths: myBaseHome\""
		if [ ! -z ${myBaseHome}   ]
		then
			exitIfNotAbsolutePath ${myBaseHome} "base home" &&
			exitIfNotADirectory ${myBaseHome} "base home" &&
			myBaseHome="$(cd ${myBaseHome} 2>/dev/null && pwd)"
		fi

		[ 3 -le $myVerbosity ] && echo [${myLevel}] "\"... checks for mandatory absolute paths: workingDir - $myWorkingDir\""
		if [ ! -z ${myWorkingDir} ]
		then
			exitIfNotAbsolutePath ${myWorkingDir} "working Directory" &&
			exitIfNotADirectory ${myWorkingDir} "working Directory" &&
			myWorkingDir="$(cd ${myWorkingDir} 2>/dev/null && pwd)"
		else
			[ 3 -le $myVerbosity ] && echo [${myLevel}] "\"... workingDir not set, set it to pwd\""
			myWorkingDir=$(pwd)
		fi

		# Convert the rest to absolute paths
		[ 3 -le $myVerbosity ] && echo [${myLevel}] "\"... convert the rest to absolute paths and check them:\""

		if [ ! -z "${myIocBootDir}" ]
		then
			[ 3 -le $myVerbosity ] && echo [${myLevel}] "\"   ... iocBootDir:\""
			isAbsolutePath ${myIocBootDir} 	       ||
				myIocBootDir=${myWorkingDir}/${myIocBootDir} && 
				[ 3 -le $myVerbosity ] && echo [${myLevel}] "iocBootDir: \"$myIocBootDir\""
			exitIfNotADirectory ${myIocBootDir}  "IOC Boot directory"
			myIocBootDir="$(cd ${myIocBootDir} 2>/dev/null && pwd)"
		fi

		if [ ! -z "${myLoggingDir}" ]
		then
			[ 3 -le $myVerbosity ] && echo [${myLevel}] "\"   ... iocLoggingDir:\""
			isAbsolutePath ${myLoggingDir}	|| myLoggingDir=${myWorkingDir}/${myLoggingDir}
			exitIfNotADirectory ${myLoggingDir}  "procServ Logging directory"
			myLoggingDir="$(cd ${myLoggingDir} 2>/dev/null && pwd)"
		fi

		if [ ! -z "${commandExecutable}" ]
		then
			[ 3 -le $myVerbosity ] && echo [${myLevel}] "\"   ... commandExecutable:\""
			isAbsolutePath ${commandExecutable} && if 1
			then 
			 	exitIfNotExecutable ${commandExecutable} "command executable"
			else
				if [ ! -z "${myIocBootDir}" ] 
				then
					if [ -f "${myIocBootDir}/${commandExecutable}" ]
					then
						exitIfNotExecutable ${myIocBootDir}/${commandExecutable} "iocBootDir extended command executable"
					fi					
				else
					commandExecutable=${myWorkingDir:+${myWorkingDir}/}${commandExecutable}				
					exitIfNotExecutable ${commandExecutable} "myWorkingDir extended command executable"
				fi		
			fi
			isAbsolutePath "${commandExecutable}"  && commandExecutable="$(cd $(dirname ${commandExecutable}) 2>/dev/null && pwd)/$(basename ${commandExecutable})"
		fi

		if [ ! -z ${myProcServExecutable} ]
		then
			[ 3 -le $myVerbosity ] && echo [${myLevel}] "\"   ... myProcServExecutable:\""
			isAbsolutePath ${myProcServExecutable} || myProcServExecutable=${myBaseHome}/${myProcServExecutable}
			exitIfNotExecutable "${myProcServExecutable}" "procServ executable"
			myProcServExecutable="$(cd $(dirname ${myProcServExecutable}) 2>/dev/null && pwd)/$(basename ${myProcServExecutable})"
		fi
	else
		echo "$scriptname: function $0 called with wrong argument only \"1\" or \"2\" is allowed  ... exiting" 1>&2
		unsets
		exit 1;
	fi
	[ 3 -le $myVerbosity ] && echo [${myLevel}] " leaving" function \"$function\" && myLevel=$(($myLevel-1))
}

# analyze IocListFile for given hostname
analyzeIocListFile()
{
	# analyzeIocListFile hostname iocEntryCounter
	#   find iocEntryCounter-nth apperance of hostname in the IOC List File
	#   and 
	#   assign found settings

	local function=analyzeIocListFile
	[ 3 -le $myVerbosity ] && myLevel=$(($myLevel+1)) && echo [${myLevel}] entering function \"$function $*\"
	local counter=0;

	# IOC list file
	# 
	# hostname | command exe  | command arguments | telnet  | telnet | logging directory | working dir a.k.a application dir | IOC boot directory | procServ exe | procServ libs | name | baseHome 
	#          |              |                   | control | log    |                   |                                   |                    |              |               |      |
	#          |              |                   | port    | port   |                   |                                   |                    |              |               |      |
	#
	[ 1 -le $myVerbosity ] && echo -e "\t" "reading in IOC list file: \"$myIocListFile\""
	[ 3 -le $myVerbosity ] && echo -e "\t" "content of IOC list file:\n" && cat $myIocListFile

	unset record
	record=$(grep $1 ${myIocListFile} | sed -e "s/ *${myIocListFileSeparator} */${myIocListFileSeparator}/g" -e "s/  */ /g" -e "s/^ *//g" | 
	awk -F "${myIocListFileSeparator}" -v no=$2 -v s=$(echo ${myIocListFileSeparator}) 'BEGIN{ctr=0}(/^[^#]/) && (!/^ *$/) {if (++ctr==no) {print $2s$3s$4s$5s$6s$7s$8s$9s$10s$11s$12}}')

	if [ ! -z "$record" ] 
	then 
		record=$(echo ${record} | sed -e "s/ (${myIocListFileSeparator}) /\1/g" | sed -e 's/^ *//g' | sed -e 's/ *$//g' )
		[ 2 -le $myVerbosity ] && echo -e "\t\t matching record found: \"$record\""
		commandExecutable=$(echo "$record" |       cut -d ${myIocListFileSeparator} -f  1 | sed -e 's/^ *//g' | sed -e 's/ *$//g' )
			[ 3 -le $myVerbosity ] && echo -e "\t\t\t assigned: commandExecutable to \"${commandExecutable}\""
		commandArguments=$(echo "$record" |        cut -d ${myIocListFileSeparator} -f  2 | sed -e 's/^ *//g' | sed -e 's/ *$//g' )
			[ 3 -le $myVerbosity ] && echo -e "\t\t\t assigned: commandArguments to \"${commandArguments}\""
		myProcServTelnetPort=$(echo "$record" |    cut -d ${myIocListFileSeparator} -f  3 | sed -e 's/^ *//g' | sed -e 's/ *$//g' )
			[ 3 -le $myVerbosity ] && echo -e "\t\t\t assigned: myProcServTelnetPort to \"${myProcServTelnetPort}\""
		myProcServTelnetLogPort=$(echo "$record" | cut -d ${myIocListFileSeparator} -f  4 | sed -e 's/^ *//g' | sed -e 's/ *$//g' )
			[ 3 -le $myVerbosity ] && echo -e "\t\t\t assigned: myProcServTelnetLogPort to \"${myProcServTelnetLogPort}\""
		myLoggingDir=$(echo "$record" |            cut -d ${myIocListFileSeparator} -f  5 | sed -e 's/^ *//g' | sed -e 's/ *$//g' )
			[ 3 -le $myVerbosity ] && echo -e "\t\t\t assigned: myLoggingDir to \"${myLoggingDir}\""
		myWorkingDir=$(echo "$record" |            cut -d ${myIocListFileSeparator} -f  6 | sed -e 's/^ *//g' | sed -e 's/ *$//g' )
			[ 3 -le $myVerbosity ] && echo -e "\t\t\t assigned: myWorkingDir to \"${myWorkingDir}\""
		myIocBootDir=$(echo "$record" |            cut -d ${myIocListFileSeparator} -f  7 | sed -e 's/^ *//g' | sed -e 's/ *$//g' )
			[ 3 -le $myVerbosity ] && echo -e "\t\t\t assigned: myIocBootDir to \"${myIocBootDir}\""
		myProcServExecutable=$(echo "$record" |    cut -d ${myIocListFileSeparator} -f  8 | sed -e 's/^ *//g' | sed -e 's/ *$//g' )
			[ 3 -le $myVerbosity ] && echo -e "\t\t\t assigned: myProcServExecutable to \"${myProcServExecutable}\""
		myLdLibPaths=$(echo "$record" |            cut -d ${myIocListFileSeparator} -f  9 | sed -e 's/^ *//g' | sed -e 's/ *$//g' )
			[ 3 -le $myVerbosity ] && echo -e "\t\t\t assigned: myLdLibPaths to \"${myLdLibPaths}\""
		myProcServIocName=$(echo "$record" |       cut -d ${myIocListFileSeparator} -f 10 | sed -e 's/^ *//g' | sed -e 's/ *$//g' )
			[ 3 -le $myVerbosity ] && echo -e "\t\t\t assigned: myProcServIocName to \"${myProcServIocName}\""
		myBaseHome=$(echo "$record" |              cut -d ${myIocListFileSeparator} -f 11 | sed -e 's/^ *//g' | sed -e 's/ *$//g' )
			[ 3 -le $myVerbosity ] && echo -e "\t\t\t assigned: myBaseHome to \"${myBaseHome}\""
		unset record
		returnValue=0
	else
		[ 2 -le $myVerbosity ] && echo -e "\t\t no matching record found\"$record\""
		returnValue=1
	fi

 	[ 3 -le $myVerbosity ] && echo [${myLevel}] " leaving" function \"$function\" && myLevel=$(($myLevel-1))
	return $returnValue;
}

#execute
execute()
{
	# main code which calls the callee
	local function=execute
	[ 3 -le $myVerbosity ] && myLevel=$(($myLevel+1)) && echo [${myLevel}] entering function \"$function $*\"

	myList=$(echo $myList | sed -e 's/"//g')
	[ 1 -le $myVerbosity ] && echo -e "\t checking for matching item in list of hostnames:" $myList

	iocEntryCounter=0

	for item in $myList
	do
		if [ "$item" = "$myHostname" ]
		then
			[ 1 -le $myVerbosity ] && echo -e "\t found match: \"$item\""

			# if IOC List File is used
			if [ ! -z "${myIocListFile}" ]
			then
				[ 1 -le $myVerbosity ] && echo -e "\t get settings from file \"${myIocListFile}\""
				iocEntryCounter=$(($iocEntryCounter +1 ))
				analyzeIocListFile $item $iocEntryCounter || continue;
				# do the second check
				[ 1 -le $myVerbosity ] && echo -e "\t "'consistency checks (2)...'
				checks 2
			fi

			[ 1 -le $myVerbosity ] && echo -e "\t calling callee ..."

			local arguments;
			counter=0; while [ $counter -lt $myVerbosityCallee ]; do arguments=${arguments}" -v"; counter=$(($counter+1)); done

			arguments=${arguments}"${myWorkingDir:+ -${myWorkingDirOpt} ${myWorkingDir}}"
			arguments=${arguments}"${myIocBootDir:+ -${myIocBootDirOpt} ${myIocBootDir}}"
			arguments=${arguments}"${commandExecutable:+ -${commandExecutableOpt} ${commandExecutable}}"
			arguments=${arguments}"${commandArguments:+ -${commandArgumentsOpt} ${commandArguments}}"
			[ -z "${myLoggingDir}" ] && arguments="${arguments}${myProcServFileLogging:+ -${myProcServFileLoggingOpt}}"
			arguments=${arguments}"${myLoggingDir:+ -${myLoggingDirOpt} ${myLoggingDir} -${myProcServFileLoggingOpt}}"
			arguments=${arguments}"${myProcServTelnetPort:+ -${myProcServTelnetPortOpt} ${myProcServTelnetPort}}"
			arguments=${arguments}"${myProcServTelnetLogPort:+ -${myProcServTelnetLogPortOpt} ${myProcServTelnetLogPort}}"
			arguments=${arguments}"${myProcServExecutable:+ -${myProcServExecutableOpt} ${myProcServExecutable}}"
			arguments=${arguments}"${myLdLibPaths:+ -${myLdLibPathsOpt} ${myLdLibPaths}}"
			arguments=${arguments}"${myBaseHome:+ -${myBaseHomeOpt} ${myBaseHome}}"
			arguments=${arguments}"${myEchoCallee:+ -${myEchoOpt} ${myEchoCallee}}"
			arguments=${arguments}"${myDebugCallee:+ -${myDebugOpt} ${myDebugCallee}}"
			arguments=${arguments}"${myProcServIocName:+ -${myProcServIocNameOpt} ${myProcServIocName}}"
			arguments=${arguments}"${calleeArguments:+ ${calleeArguments}}"

			${myEcho} export HOSTNAME=${HOSTNAME:=${myHostname}}
			eval ${myEcho} ${myCalleeScript} ${arguments}

			#multiple IOC are allowed?
			if [ -z "$multipleIocs" ]
			then
				break
			else
				[ 1 -le $myVerbosity ] && echo -e "\t since multiple IOCs are allowed: heat $iocEntryCounter finished"
			fi
		fi
	done
	[ 3 -le $myVerbosity ] && echo [${myLevel}] " leaving" function \"$function\" && myLevel=$(($myLevel-1))

	[ 3 -le $myVerbosity ] && echo [${myLevel}] " leaving" function \"execute\" && myLevel=$(($myLevel-1))
}

# function help
help()
{
	usage ;
	unsets;
	exit 0;
}

### main
#init functions
	predefines
	specificPredefines

#eval options
		oldIFS=$IFS #store old IFS
		IFS="|"     #change IFS to handle quotes when passing args to functions
	evalOptions $(echo "$*""$IFS" | sed -e "s/\(-[a-zA-Z0-9]$IFS\)\([^-]*\)\($IFS\)/\1\"\2\"\3/g")
		IFS=$oldIFS #reset IFS

#call Help if help option is set
	[ ! -z "$myHelp" ] && help;

#load helper functions
	if [ -f "$myExitIfHelperScript" ]
	then
		. ${myExitIfHelperScript}
	else
		echo "$scriptname: helper script \`$myExitIfHelperScript' does not exists ... exiting" 1>&2
		unsets
		exit 1;
	fi

	[ 1 -le $myVerbosity ] && echo $scriptname ...
        [ 2 -le $myVerbosity ] && dump;

# debug settings
	[ ! -z $myDebug ] && set -x -v

#if log port not set, then logport = port + 1
	[ -z "$myProcServTelnetLogPort" ] && myProcServTelnetLogPort=$((${myProcServTelnetPort} + 1))

	[ 1 -le $myVerbosity ] && echo -e "\t "'consistency checks (1)...'

#checks first round
	checks 1

# IOC list file
	# - get list of hostnames from IOC list file
	if [ ! -z "${myIocListFile}" ]
	then
		[ 1 -le $myVerbosity ] && echo -e "\t reading in IOC list file: \"$myIocListFile\""
		[ 3 -le $myVerbosity ] && echo -e "\t content of IOC list file:\n" && cat $myIocListFile
		myList=$(sed -e "s/ *${myIocListFileSeparator} */${myIocListFileSeparator}/g" -e "s/  */ /g" -e "s/^ *//g" ${myIocListFile} |\
			 awk -F "${myIocListFileSeparator}" '(/^[^#]/) && (!/^ *$/) {print $1}')
		myListLength=$(echo $myList | wc -w)
# 		myList=$(grep -v '^#' ${myIocListFile} | sed -e "s/  */ /g" | cut -d "${myIocListFileSeparator}" -f 1 )
		[ 2 -le $myVerbosity ] && echo -e "\t\t" "found hostnames: "\`$myList\'
	fi

# if IOC list is not used/set, do the second round of check
	if [ -z "${myIocListFile}" ]
	then
		[ 1 -le $myVerbosity ] && echo -e "\t"' consistency checks (2) ...'
 		checks 2;
	fi


# execute callee caller, etc.. 
	execute
# clean
	unsets
exit 0
