<?php

/*
	Phoronix Test Suite
	URLs: http://www.phoronix.com, http://www.phoronix-test-suite.com/
	Copyright (C) 2009 - 2022, Phoronix Media
	Copyright (C) 2009 - 2022, Michael Larabel

	This program is free software; you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation; either version 3 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

class phoromatic extends pts_module_interface
{
	const module_name = 'Phoromatic Client';
	const module_version = '1.2.0';
	const module_description = 'The Phoromatic client is used for connecting to a Phoromatic server (Phoromatic.com or a locally run server) to facilitate the automatic running of tests, generally across multiple test nodes in a routine manner. For more details visit http://www.phoromatic.com/. This module is intended to be used with Phoronix Test Suite 5.2+ clients and servers.';
	const module_author = 'Phoronix Media';

	private static $account_id = null;
	private static $server_address = null;
	private static $server_http_port = null;
	private static $use_https = false;
	private static $server_ws_port = null;
	private static $is_running_as_phoromatic_node = false;
	private static $log_file = null;
	private static $limit_network_communication = false;
	private static $system_id = null;
	private static $server_core_version = 0;
	private static $progressive_result_uploads = false;

	private static $p_save_identifier = null;
	private static $p_schedule_id = null;
	private static $p_trigger_id = null;
	private static $benchmark_ticket_id = null;
	private static $in_stress_mode = false;
	private static $has_run_server_setup_func = false;

	private static $test_run_manager = null;

	public static function module_info()
	{
		return 'The Phoromatic module contains the client support for interacting with Phoromatic and Phoromatic Tracker services.';
	}
	public static function user_commands()
	{
		return array('connect' => 'run_connection', 'explore' => 'explore_network', 'upload_result' => 'upload_unscheduled_result', 'set_root_admin_password' => 'set_root_admin_password', 'list_results' => 'recent_phoromatic_server_results', 'clone' => 'clone_phoromatic_server_result', 'export_results_for_account_schedules' => 'generate_export_result_schedule_dump');
	}
	public static function upload_unscheduled_result($args)
	{
		$server_setup = self::setup_server_addressing($args);

		if(!$server_setup)
			return false;

		$uploads = 0;
		foreach($args as $arg)
		{
			if(pts_types::is_result_file($arg))
			{
				$uploads++;
				echo PHP_EOL . 'Uploading: ' . $arg . PHP_EOL;
				$result_file = pts_types::identifier_to_object($arg);
				$server_response = self::upload_test_result($result_file);
				$server_response = json_decode($server_response, true);
				if(isset($server_response['phoromatic']['response']))
					echo '   Result Uploaded' . PHP_EOL;
				else
					echo '   Upload Failed' . PHP_EOL;
			}
		}

		if($uploads == 0)
			echo PHP_EOL . 'No Result Files Found To Upload.' . PHP_EOL;
	}
	public static function set_root_admin_password()
	{
		phoromatic_server::prepare_database();
		$root_admin_pw = phoromatic_server::read_setting('root_admin_pw');

		// DISABLED THE INITIAL CHECKING CODE SINCE REALLY WAS SILLY SINCE IF THEY ALREADY HAVE TERMINAL ACCESS, THEY CAN ACCESS FILE ANYWAY....
		if(false && $root_admin_pw != null)
		{
			do
			{
				$check_root_pw = pts_user_io::prompt_user_input('Please enter the existing root-admin password');
			}
			while(hash('sha256', 'PTS' . $check_root_pw) != $root_admin_pw);
		}

		echo PHP_EOL . 'The new root-admin password must be at least six characters long.' . PHP_EOL;
		do
		{
			$new_root_pw = pts_user_io::prompt_user_input('Please enter the new root-admin password');
		}
		while(strlen($new_root_pw) < 6);

		$new_root_pw = hash('sha256', 'PTS' . $new_root_pw);
		$root_admin_pw = phoromatic_server::save_setting('root_admin_pw', $new_root_pw);
	}
	public static function generate_export_result_schedule_dump($r)
	{
		phoromatic_server::prepare_database();
		if(isset($r[0]) && !empty($r[0]))
		{
			phoromatic_server::generate_result_export_dump($r[0]);
		}
	}
	public static function phoromatic_server_supports_https($address, $port)
	{
		// Use 3 second timeout for HTTPS request to ensure not wasting too much time...
		$response = pts_network::http_get_contents('https://' . $address . ':' . $port . '/server.php?phoromatic_info', false, false, false, false, 3);

		if(!empty($response) && ($response = json_decode($response, true)) != null && isset($response['pts']))
		{
			// Successfully obtaining data via HTTPS
			return true;
		}
		return false;
	}
	public static function explore_network()
	{
		pts_client::$display->generic_heading('Phoromatic Servers');
		$archived_servers = pts_client::available_phoromatic_servers();

		$server_count = 0;
		foreach($archived_servers as $archived_server)
		{
			$supports_https = $archived_server['protocol'] == 'https'; // Previous code: self::phoromatic_server_supports_https($archived_server['ip'], $archived_server['http_port']);
			$response = pts_network::http_get_contents(($supports_https ? 'https://' : 'http://') . $archived_server['ip'] . ':' . $archived_server['http_port'] . '/server.php?phoromatic_info');

			if(!empty($response))
			{
				$response = json_decode($response, true);
				if($response && isset($response['pts']))
				{
					$server_count++;
					echo PHP_EOL . 'IP: ' . $archived_server['ip'] . PHP_EOL;
					echo 'HTTP PORT: ' . $archived_server['http_port'] . PHP_EOL;
					echo 'PROTOCOL: ' . ($supports_https ? 'HTTPS': 'HTTP') . PHP_EOL;
					echo 'WEBSOCKET PORT: ' . $response['ws_port'] . PHP_EOL;
					echo 'SERVER: ' . $response['http_server'] . PHP_EOL;
					echo 'PHORONIX TEST SUITE: ' . $response['pts'] . ' [' . $response['pts_core'] . ']' . PHP_EOL;

					// TODO XXX fix/finish below code...
					if(false && ($ws = new phoromatic_client_comm_ws($archived_server['ip'], $response['ws_port'])))
					{
						// Query the WebSocket Server for some Phoromatic Server details
						$s = $ws->send(array('phoromatic' => array('event' => 'pts-version')));
						$s = $ws->send(array('phoromatic' => array('event' => 'download-cache')));
						$r = $ws->receive_until('download-cache');
						var_dump($r);
					}
					else
					{

						// Provide some other server info via HTTP

						$repo = pts_network::http_get_contents(($supports_https ? 'https://' : 'http://') . $archived_server['ip'] . ':' . $archived_server['http_port'] . '/download-cache.php?repo');
						echo 'DOWNLOAD CACHE: ';
						if(!empty($repo))
						{
							$repo = json_decode($repo, true);
							if($repo && isset($repo['phoronix-test-suite']['download-cache']))
							{
								$total_file_size = 0;
								foreach($repo['phoronix-test-suite']['download-cache'] as $file_name => $inf)
								{
									$total_file_size += $repo['phoronix-test-suite']['download-cache'][$file_name]['file_size'];
								}
								echo count($repo['phoronix-test-suite']['download-cache']) . ' FILES / ' . round($total_file_size / 1000000) . ' MB CACHE SIZE';
							}
						}
						else
						{
							echo 'N/A';
						}
					}

					echo PHP_EOL;

					$repo = pts_network::http_get_contents(($supports_https ? 'https://' : 'http://') . $archived_server['ip'] . ':' . $archived_server['http_port'] . '/openbenchmarking-cache.php?repos');
					echo 'SUPPORTED OPENBENCHMARKING.ORG REPOSITORIES:' . PHP_EOL;
					if(!empty($repo))
					{
						$repo = json_decode($repo, true);
						if($repo && is_array($repo['repos']))
						{
							foreach($repo['repos'] as $data)
							{
								echo '      ' . $data['title'] . ' - Last Generated: ' . date('d M Y H:i', $data['generated']) . PHP_EOL;
							}
						}
					}
					else
					{
						echo '      N/A' . PHP_EOL;
					}
				}
			}
		}

		if($server_count == 0)
		{
			echo PHP_EOL . 'No Phoromatic Servers detected.' . PHP_EOL . PHP_EOL;
		}
	}
	protected static function tick_thread($force_manual_poll = false)
	{
		static $last_phoromatic_log = 0;

		while(true)
		{
			$j = array();

			$log_size = pts_client::$pts_logger->get_log_file_size();
			if($log_size != $last_phoromatic_log)
			{
				$phoromatic_log = file_get_contents(pts_client::$pts_logger->get_log_file_location());
				$last_phoromatic_log = $log_size;
				$j['phoromatic']['client-log'] = $phoromatic_log;
			}

			foreach(phodevi::supported_sensors() as $sensor)
			{
				$j['phoromatic']['stats']['sensors'][phodevi::sensor_name($sensor)] = array('value' => phodevi::read_sensor($sensor), 'unit' => phodevi::read_sensor_unit($sensor));
			}

			$j['phoromatic']['stats']['uptime'] = ceil(phodevi::system_uptime() / 60);

			$server_response = phoromatic::upload_to_remote_server(array(
					'r' => 'tick',
					'j' => json_encode($j),
					));

			if($force_manual_poll)
			{
				return;
			}

			$server_response = json_decode($server_response, true);
			if($server_response && isset($server_response['phoromatic']['tick_thread']))
			{
				switch($server_response['phoromatic']['tick_thread'])
				{
					case 'reboot':
						phodevi::reboot();
						break;
					case 'halt-testing':
						touch(PTS_USER_PATH . 'halt-testing');
						break;
				}
			}

			// Randomize the thread work a little bit to ensure not hitting the systems at the same time
			sleep(rand(60, 90));
		}
	}
	protected static function upload_to_remote_server($to_post, $server_address = null, $server_http_port = null, $account_id = null, $try_https = false)
	{
		static $last_communication_minute = null;
		static $communication_attempts = 0;

		if($last_communication_minute == date('i') && $communication_attempts > 8)
		{
				// Something is wrong, Phoromatic shouldn't be communicating with server more than four times a minute
				return false;
		}
		else
		{
			if(date('i') != $last_communication_minute)
			{
				$last_communication_minute = date('i');
				$communication_attempts = 0;
			}

			$communication_attempts++;
		}

		if($server_address == null && self::$server_address != null)
		{
			$server_address = self::$server_address;
		}
		if($server_http_port == null && self::$server_http_port != null)
		{
			$server_http_port = self::$server_http_port;
		}
		if($account_id == null && self::$account_id != null)
		{
			$account_id = self::$account_id;
		}

		if($try_https)
		{
			$try_https = self::phoromatic_server_supports_https($server_address, $server_http_port);
		}

		$to_post['aid'] = $account_id;
		$to_post['pts'] = PTS_VERSION;
		$to_post['pts_core'] = PTS_CORE_VERSION;
		$to_post['gsid'] = defined('PTS_GSID') ? PTS_GSID : null;
		$to_post['lip'] = phodevi::read_property('network', 'ip');
		$to_post['h'] = phodevi::system_hardware(true);
		$to_post['nm'] = phodevi::read_property('network', 'mac-address');
		$to_post['nw'] = implode(', ', pts_network::get_network_wol());
		$to_post['s'] = phodevi::system_software(true);
		$to_post['n'] = phodevi::read_property('system', 'hostname');
		$to_post['pp'] = json_encode(phodevi::read_all_properties());
		$to_post['msi'] = PTS_MACHINE_SELF_ID;
		return pts_network::http_upload_via_post((self::$use_https || $try_https ? 'https://' : 'http://') . $server_address . ':' . $server_http_port .  '/phoromatic.php', $to_post, false);
	}
	protected static function update_system_status($current_task, $estimated_time_remaining = 0, $percent_complete = 0, $for_schedule = null, $estimate_to_next_comm = 0)
	{
		static $last_msg = null;

		// Avoid an endless flow of "idling" messages, etc
		if($current_task != $last_msg)
			pts_client::$pts_logger && pts_client::$pts_logger->log($current_task);
		$last_msg = $current_task;

		if(self::$limit_network_communication)
		{
			static $last_comm_time = 0;
			if(time() > ($last_comm_time + 800 + rand(0, 180)))
			{
				// It's been at least half hour since last update, so report in state...
				$last_comm_time = time();
			}
			else
			{
				return;
			}
		}

		return phoromatic::upload_to_remote_server(array(
				'r' => 'update_system_status',
				'a' => $current_task,
				'time' => $estimated_time_remaining,
				'pc' => $percent_complete,
				'sched' => (!empty($for_schedule) ? $for_schedule : self::$p_schedule_id),
				'bid' => (!empty(self::$benchmark_ticket_id) ? self::$benchmark_ticket_id : 0),
				'o' => $estimate_to_next_comm
				));
	}
	protected static function set_server_info($address, $port, $account_id)
	{
		self::$server_address = $address;
		self::$server_http_port = $port;
		self::$account_id = $account_id;
		self::$use_https = self::phoromatic_server_supports_https(self::$server_address, self::$server_http_port);
	}
	protected static function setup_server_addressing($server_string = null)
	{
		self::$has_run_server_setup_func = true;

		if(isset($server_string[0]) && strpos($server_string[0], '/', strpos($server_string[0], ':')) > 6)
		{
			pts_client::$pts_logger && pts_client::$pts_logger->log('Attempting to connect to Phoromatic Server: ' . $server_string[0]);
			self::set_server_info(substr($server_string[0], 0, strpos($server_string[0], ':')), substr($server_string[0], strlen(substr($server_string[0], 0, strpos($server_string[0], ':'))) + 1, -1 - strlen(substr($server_string[0], strrpos($server_string[0], '/') + 1))), substr($server_string[0], strrpos($server_string[0], '/') + 1));
			pts_client::$display->generic_heading('Server IP: ' . self::$server_address . PHP_EOL . 'Server HTTP Port: ' . self::$server_http_port . PHP_EOL . 'Account ID: ' . self::$account_id);
			pts_client::register_phoromatic_server(self::$server_address, self::$server_http_port);
		}
		else if(($last_server = trim(pts_module::read_file('last-phoromatic-server'))) && !empty($last_server))
		{
			pts_client::$pts_logger && pts_client::$pts_logger->log('Attempting to connect to last server connection: ' . $last_server);
			$last_account_id = substr($last_server, strrpos($last_server, '/') + 1);
			$last_server_address = substr($last_server, 0, strpos($last_server, ':'));
			$last_server_http_port = substr($last_server, strlen($last_server_address) + 1, -1 - strlen($last_account_id));
			pts_client::$pts_logger && pts_client::$pts_logger->log('Last Server IP: ' . $last_server_address . ' Last Server HTTP Port: ' . $last_server_http_port . ' Last Account ID: ' . $last_account_id);

			for($i = 0; $i < 10; $i++)
			{
				$server_response = phoromatic::upload_to_remote_server(array(
					'r' => 'ping',
					), $last_server_address, $last_server_http_port, $last_account_id, true);

				$server_response = json_decode($server_response, true);
				if($server_response && isset($server_response['phoromatic']['ping']))
				{
					self::set_server_info($last_server_address, $last_server_http_port, $last_account_id);
					pts_client::$pts_logger && pts_client::$pts_logger->log('Phoromatic Server connection restored');
					pts_client::register_phoromatic_server(self::$server_address, self::$server_http_port);
					break;
				}
				else
				{
					pts_client::$pts_logger && pts_client::$pts_logger->log('Phoromatic Server connection failed');
					sleep((12 * ($i + 1)));
				}
			}
		}

		if(self::$server_address == null)
		{
			$archived_servers = pts_client::available_phoromatic_servers();
			if(!empty($archived_servers))
			{
				pts_client::$pts_logger && pts_client::$pts_logger->log('Attempting to auto-discover Phoromatic Servers');
				self::attempt_phoromatic_server_auto_discover($archived_servers);
			}
		}

		if(self::$server_address == null || self::$server_http_port == null || self::$account_id == null)
		{
			pts_client::$pts_logger && pts_client::$pts_logger->log('Phoromatic Server connection setup failed');
			echo PHP_EOL . 'You must pass the Phoromatic Server information as an argument to phoromatic.connect, or otherwise configure your network setup.' . PHP_EOL . '      e.g. phoronix-test-suite phoromatic.connect 192.168.1.2:5555/I0SSJY' . PHP_EOL . PHP_EOL;

			if(PTS_IS_DAEMONIZED_SERVER_PROCESS && !empty($archived_servers))
			{
				echo 'The Phoromatic client appears to be running as a system service/daemon so will attempt to continue auto-polling to find the Phoromatic Server.' . PHP_EOL . PHP_EOL;

				$success = false;
				do
				{
					pts_client::$pts_logger && pts_client::$pts_logger->log('Will auto-poll connected servers every 90 seconds looking for a claim by a Phoromatic Server');
					sleep(90);
					$success = self::attempt_phoromatic_server_auto_discover($archived_servers);
				}
				while($success == false);
			}
			else
			{
				return false;
			}
		}

		return true;
	}
	protected static function attempt_phoromatic_server_auto_discover(&$phoromatic_servers)
	{
		foreach($phoromatic_servers as &$archived_server)
		{
			pts_client::$pts_logger && pts_client::$pts_logger->log('Attempting to auto-discover Phoromatic Server on: ' . $archived_server['ip'] . ': ' . $archived_server['http_port']);
			$server_response = phoromatic::upload_to_remote_server(array(
				'r' => 'ping',
				), $archived_server['ip'], $archived_server['http_port'], null, true);

			$server_response = json_decode($server_response, true);
			if($server_response && isset($server_response['phoromatic']['account_id']))
			{
				self::set_server_info($archived_server['ip'], $archived_server['http_port'], $server_response['phoromatic']['account_id']);
				return true;
			}
		}

		return false;
	}
	protected static function setup_system_environment()
	{
		if(is_writable('/boot/grub/grubenv') && pts_client::executable_in_path('grub-editenv'))
		{
			// In case system fails or reboots in process and don't want to hang on GRUB recordfail
			shell_exec('grub-editenv /boot/grub/grubenv unset recordfail 2>&1');
		}
	}
	public static function run_connection($args)
	{
		if(pts_client::create_lock(PTS_USER_PATH . 'phoromatic_lock') == false)
		{
			trigger_error('Phoromatic is already running.', E_USER_ERROR);
			return false;
		}
		define('PHOROMATIC_PROCESS', true);

		if(pts_client::$pts_logger == false)
		{
			pts_client::$pts_logger = new pts_logger();
		}
		pts_client::$pts_logger->log(pts_core::program_title() . ' [' . PTS_CORE_VERSION . '] starting Phoromatic client');

		if(phodevi::system_uptime() < 60)
		{
			echo 'PHOROMATIC: Sleeping for 60 seconds as system freshly started.' . PHP_EOL;
			pts_client::$pts_logger->log('Sleeping for 60 seconds as system freshly started');
			sleep(60);
		}

		$server_setup = self::setup_server_addressing($args);
		//$http_comm = new phoromatic_client_comm_http();

		/*
		if(!$server_setup)
		{
			if(getenv('PTS_NO_REBOOT_ON_NETWORK_FAILURE') == false && PTS_IS_DAEMONIZED_SERVER_PROCESS)
			{
				phodevi::reboot();
			}

			return false;
		}
		*/

		$times_failed = 0;
		$has_success = false;
		$do_exit = false;
		$just_started = true;

		self::setup_system_environment();
		pts_client::$pts_logger->log('SYSTEM HARDWARE: ' . phodevi::system_hardware(true));
		pts_client::$pts_logger->log('SYSTEM SOFTWARE: ' . phodevi::system_software(true));

		while($do_exit == false)
		{
			$server_response = phoromatic::upload_to_remote_server(array(
				'r' => 'start',
				));

			if($server_response == false)
			{
				$times_failed++;

				pts_client::$pts_logger->log('Server response failed');

				if($times_failed >= 2)
				{
					trigger_error('Communication with server failed.', E_USER_ERROR);

					if(PTS_IS_DAEMONIZED_SERVER_PROCESS == false && $times_failed > 5)
					{
						return false;
					}
					else if(PTS_IS_DAEMONIZED_SERVER_PROCESS && $times_failed > 10)
					{
						if(getenv('PTS_NO_REBOOT_ON_NETWORK_FAILURE') == false)
						{
							phodevi::reboot();
						}
					}
				}
			}
			else if(substr($server_response, 0, 1) == '[')
			{
				// Likely a notice/warning from server
				echo PHP_EOL . substr($server_response, 0, strpos($server_response, PHP_EOL)) . PHP_EOL;
			}
			else if(substr($server_response, 0, 1) == '{')
			{
				$times_failed = 0;
				$json = json_decode($server_response, true);

				if($has_success == false)
				{
					$has_success = true;
					pts_module::save_file('last-phoromatic-server', self::$server_address . ':' . self::$server_http_port . '/' . self::$account_id);
				}

				if($json != null)
				{
					if(isset($json['phoromatic']['error']) && !empty($json['phoromatic']['error']))
					{
						trigger_error($json['phoromatic']['error'], E_USER_ERROR);
					}
					if(isset($json['phoromatic']['response']) && !empty($json['phoromatic']['response']))
					{
						echo PHP_EOL . $json['phoromatic']['response'] . PHP_EOL;
					}
					if(isset($json['phoromatic']['system_id']) && !empty($json['phoromatic']['system_id']))
					{
						if(self::$system_id != $json['phoromatic']['system_id'])
						{
							// The Phoromatic server/account's system ID for this given system
							self::$system_id = $json['phoromatic']['system_id'];

							// Save the system ID to text file if it's useful for other purposes...
							pts_module::set_option('system_id', self::$system_id);
						}
					}
					if(isset($json['phoromatic']['server_core_version']) && !empty($json['phoromatic']['server_core_version']))
					{
						if(self::$server_core_version != $json['phoromatic']['server_core_version'])
						{
							// The PTS core version of the Phoromatic Server
							self::$server_core_version = $json['phoromatic']['server_core_version'];
						}
					}
				}

				self::$limit_network_communication = isset($json['phoromatic']['settings']['LimitNetworkCommunication']) && pts_strings::string_bool($json['phoromatic']['settings']['LimitNetworkCommunication']);
				if(self::$limit_network_communication)
				{
					// Sleep to ensure network communication is somewhat random in case all systems started at same time
					sleep(rand(0, 20));
				}

				if($just_started)
				{
					if(PTS_IS_DAEMONIZED_SERVER_PROCESS && !self::$limit_network_communication && function_exists('pcntl_fork'))
					{
						$pid = pcntl_fork();
						if($pid == 0)
						{
							// Start the tick thread
							self::tick_thread();
						}
					}

					self::upload_test_install_manifest();
					$just_started = false;
				}

				pts_tests::clear_extra_env_vars();
				if(isset($json['phoromatic']['pre_set_sys_env_vars']) && !empty($json['phoromatic']['pre_set_sys_env_vars']))
				{
					// pre_set_sys_env_vars was added during PTS 5.8 development
					// Sets environment variables on client as specified via the Phoromatic Server's systems page
					foreach(explode(';', $json['phoromatic']['pre_set_sys_env_vars']) as $i => $v_string)
					{
						$var = explode('=', $v_string);
						if(count($var) == 2)
						{
							putenv($var[0] . '=' . $var[1]);
							pts_tests::add_extra_env_var($var[0], $var[1]);
						}
					}
				}

				$task = isset($json['phoromatic']['task']) ? $json['phoromatic']['task'] : null;

				if($task != 'idle')
				{
					pts_client::$pts_logger->log("Received " . $task . " command");
				}

				switch($task)
				{
					case 'install':
						phoromatic::update_system_status('Installing Tests');
						$phoromatic_suite = new pts_test_suite($json['phoromatic']['test_suite']);
						pts_test_installer::standard_install($phoromatic_suite, false, true);
						break;
					case 'benchmark':

						// Make sure all latest tests are available
						pts_openbenchmarking::refresh_repository_lists(null, true);

						$benchmark_timer = time();
						self::$is_running_as_phoromatic_node = true;
						$phoromatic_suite = new pts_test_suite($json['phoromatic']['test_suite']);
						$phoromatic_results_identifier = $json['phoromatic']['trigger_id'];
						self::$p_save_identifier = $json['phoromatic']['save_identifier'];
						self::$p_schedule_id = isset($json['phoromatic']['schedule_id']) ? $json['phoromatic']['schedule_id'] : false;
						self::$p_trigger_id = $json['phoromatic']['trigger_id'];
						self::$benchmark_ticket_id = isset($json['phoromatic']['benchmark_ticket_id']) ? $json['phoromatic']['benchmark_ticket_id'] : null;
						phoromatic::update_system_status('Running Benchmarks For: ' . self::$p_save_identifier);
						// PTS 10.8 result viewer should deal fine with binary logs so can just upload by default there...
						pts_client::$skip_log_file_type_checks = self::$server_core_version >= 10800 || (isset($json['phoromatic']['settings']['AllowAnyDataForLogFiles']) && pts_strings::string_bool($json['phoromatic']['settings']['AllowAnyDataForLogFiles']));
						self::$progressive_result_uploads = isset($json['phoromatic']['settings']['ProgressiveResultUploads']) && pts_strings::string_bool($json['phoromatic']['settings']['ProgressiveResultUploads']);
						pts_module::set_option('skip_log_file_type_checks', (pts_client::$skip_log_file_type_checks ? 1 : 0));

						if(pts_strings::string_bool($json['phoromatic']['settings']['RunInstallCommand']))
						{
							if(isset($json['phoromatic']['pre_install_set_context']))
							{
								phoromatic::set_user_context($json['phoromatic']['pre_install_set_context'], self::$p_trigger_id, self::$p_schedule_id, 'PRE_INSTALL');
							}

							pts_test_installer::standard_install($phoromatic_suite, pts_strings::string_bool($json['phoromatic']['settings']['ForceInstallTests']), true);
							if(isset($json['phoromatic']['post_install_set_context']))
							{
								phoromatic::set_user_context($json['phoromatic']['post_install_set_context'], self::$p_trigger_id, self::$p_schedule_id, 'POST_INSTALL');
							}
						}
						$env_vars = isset($json['phoromatic']['environment_variables']) ? pts_strings::parse_value_string_vars($json['phoromatic']['environment_variables']) : array();
						$is_stress_run = isset($env_vars['PTS_CONCURRENT_TEST_RUNS']) && $env_vars['PTS_CONCURRENT_TEST_RUNS'] > 1;

						// Do the actual running
						phodevi::clear_cache();
						$original_env_var_overrides = pts_env::get_overrides();
						$original_pts_modules = pts_module_manager::attached_modules();

						if(isset($json['phoromatic']['settings']['GlobalEnvironmentVariables']) && !empty($json['phoromatic']['settings']['GlobalEnvironmentVariables']))
						{
							// The global environment variables set on the Phoromatic Settings page, rather than individual schedule/benchmark ticket specific env vars
							pts_env::set_array(pts_strings::parse_value_string_vars($json['phoromatic']['settings']['GlobalEnvironmentVariables']));
						}

						if(!empty($env_vars))
						{
							pts_env::set_array($env_vars);
						}

						if($is_stress_run)
						{
							self::$test_run_manager = new pts_stress_run_manager(array(
								'UploadResults' => false,
								'SaveResults' => false,
								'PromptForTestDescription' => false,
								'RunAllTestCombinations' => false,
								'PromptSaveName' => false,
								'PromptForTestIdentifier' => false,
								'OpenBrowser' => false
								), true);

							if(self::$test_run_manager->initial_checks($phoromatic_suite, 'SHORT'))
							{
								if(self::$test_run_manager->load_tests_to_run($phoromatic_suite))
								{
									self::$test_run_manager->action_on_stress_log_set(array('phoromatic', 'upload_stress_log_sane'));
									self::$in_stress_mode = self::$p_save_identifier;
									self::$test_run_manager->multi_test_stress_run_execute($env_vars['PTS_CONCURRENT_TEST_RUNS'], $env_vars['TOTAL_LOOP_TIME']);
									self::$in_stress_mode = false;
									self::upload_stress_log(self::$test_run_manager->get_stress_log());
								}
							}
							self::$benchmark_ticket_id = null;
							break;
						}
						else
						{
							self::$test_run_manager = new pts_test_run_manager(array(
							'UploadResults' => (isset($json['phoromatic']['settings']['UploadResultsToOpenBenchmarking']) && pts_strings::string_bool($json['phoromatic']['settings']['UploadResultsToOpenBenchmarking'])),
							'SaveResults' => true,
							'RunAllTestCombinations' => false,
							'OpenBrowser' => false
							), true);
						}

						if(self::$test_run_manager->initial_checks($phoromatic_suite, 'SHORT'))
						{
							// Load the tests to run
							if(self::$test_run_manager->load_tests_to_run($phoromatic_suite))
							{
								phoromatic::update_system_status('Tests In Run Queue: ' . implode(', ', self::$test_run_manager->get_tests_to_run_identifiers()));
								if(isset($json['phoromatic']['pre_run_set_context']))
								{
									phoromatic::set_user_context($json['phoromatic']['pre_run_set_context'], self::$p_trigger_id, self::$p_schedule_id, 'PRE_RUN');
								}

								if(isset($json['phoromatic']['settings']['UploadResultsToOpenBenchmarking']) && pts_strings::string_bool($json['phoromatic']['settings']['UploadResultsToOpenBenchmarking']))
								{
									self::$test_run_manager->auto_upload_to_openbenchmarking();
									pts_openbenchmarking_client::override_client_setting('UploadSystemLogsByDefault', pts_strings::string_bool($json['phoromatic']['settings']['UploadSystemLogs']));
								}

								// Save results?

								// Run the actual tests
								self::$test_run_manager->auto_save_results(self::$p_save_identifier, $phoromatic_results_identifier, (isset($json['phoromatic']['test_description']) ? $json['phoromatic']['test_description'] : 'A Phoromatic run.'));
								self::$test_run_manager->pre_execution_process();
								self::$test_run_manager->call_test_runs();

								phoromatic::update_system_status('Benchmarks Completed For: ' .  self::$p_save_identifier);
								self::$test_run_manager->post_execution_process();
								$elapsed_benchmark_time = time() - $benchmark_timer;

								// Handle uploading data to server
								$result_file = new pts_result_file(self::$test_run_manager->get_file_name());
								$upload_system_logs = pts_strings::string_bool($json['phoromatic']['settings']['UploadSystemLogs']);
								pts_module::set_option('upload_system_logs', ($upload_system_logs ? 1 : 0));
								$upload_install_logs = isset($json['phoromatic']['settings']['UploadInstallLogs']) && pts_strings::string_bool($json['phoromatic']['settings']['UploadInstallLogs']);
								pts_module::set_option('upload_install_logs', ($upload_install_logs ? 1 : 0));
								$upload_run_logs = isset($json['phoromatic']['settings']['UploadRunLogs']) && pts_strings::string_bool($json['phoromatic']['settings']['UploadRunLogs']);
								pts_module::set_option('upload_run_logs', ($upload_run_logs ? 1 : 0));
								$server_response = self::upload_test_result($result_file, $upload_system_logs, self::$p_schedule_id, self::$p_save_identifier, self::$p_trigger_id, $elapsed_benchmark_time, self::$benchmark_ticket_id);
								//pts_client::$pts_logger->log('DEBUG RESPONSE MESSAGE: ' . $server_response);
								if(!pts_strings::string_bool($json['phoromatic']['settings']['ArchiveResultsLocally']))
								{
									pts_results::remove_saved_result_file(self::$test_run_manager->get_file_name());
								}
							}

							if(isset($json['phoromatic']['post_run_set_context']))
							{
								phoromatic::set_user_context($json['phoromatic']['post_run_set_context'], self::$p_trigger_id, self::$p_schedule_id, 'POST_RUN');
							}
						}
						self::$p_schedule_id = null;
						self::$is_running_as_phoromatic_node = false;
						self::$benchmark_ticket_id = null;

						// Restore any environment variables that may have been set within process / overridden
						if(!empty($original_env_var_overrides))
						{
							pts_env::set_array($original_env_var_overrides, true);
							$original_env_var_overrides = null;
						}
						// Unload any modules that were loaded just during this benchmarking run (i.e. by a passed environment variable from Phoromatic)
						pts_module_manager::detach_extra_modules($original_pts_modules);
						break;
					case 'reboot':
						echo PHP_EOL . 'Phoromatic received a remote command to reboot.' . PHP_EOL;
						phoromatic::update_system_status('Attempting System Reboot');
						self::tick_thread(true);
						phodevi::reboot();
						break;
					case 'shutdown-if-supports-wake':
						$supports_wol = false;
						foreach(pts_network::get_network_wol() as $net_device)
						{
							if(strpos($net_device, 'g') !== false)
							{
								$supports_wol = true;
								break;
							}
						}
						if(!$supports_wol)
							break;
					case 'shutdown':
						if(isset($json['phoromatic']['client_update_script']) && !empty($json['phoromatic']['client_update_script']))
						{
							self::run_client_update_script($json['phoromatic']['client_update_script']);
							sleep(10);
						}

						echo PHP_EOL . 'Phoromatic received a remote command to shutdown.' . PHP_EOL;
						phoromatic::update_system_status('Attempting System Shutdown');
						self::tick_thread(true);
						phodevi::shutdown();
						break;
					case 'maintenance':
						echo PHP_EOL . 'Idling, system maintenance mode set by Phoromatic Server.' . PHP_EOL;
						phoromatic::update_system_status('Maintenance Mode' . self::check_for_separate_pts_thread_process());
						sleep(60);
						break;
					case 'idle':
						if(isset($json['phoromatic']['client_update_script']) && !empty($json['phoromatic']['client_update_script']))
						{
							self::run_client_update_script($json['phoromatic']['client_update_script']);
						}
						//echo PHP_EOL . 'Idling, waiting for task.' . PHP_EOL;
						phoromatic::update_system_status('Idling, Waiting For Task' . self::check_for_separate_pts_thread_process());
						break;
					case 'exit':
						echo PHP_EOL . 'Phoromatic received a remote command to exit.' . PHP_EOL;
						phoromatic::update_system_status('Exiting Phoromatic');
						self::tick_thread(true);
						$do_exit = true;
						break;
				}
			}

			if(!$do_exit)
			{
				if($server_response == false)
					sleep(rand(10, 30));
				else if(self::$limit_network_communication)
					sleep(rand(60, 240));
				else
					sleep(60);
			}
		}

		pts_client::release_lock(PTS_USER_PATH . 'phoromatic_lock');
	}
	private static function check_for_separate_pts_thread_process()
	{
		$report = null;
		$log_file = pts_logger::default_log_file_path() . 'phoronix-test-suite-benchmark.log';
		if(is_file($log_file) && filemtime($log_file) > (time() - 1200))
		{
			$log_file = pts_file_io::file_get_contents($log_file);
			$log_file = substr($log_file, strrpos($log_file, PHP_EOL) + 1);
			if(($x = strpos($log_file, ']')) !== false)
			{
				$log_file = substr($log_file, ($x + 1));
			}
			$report .= '; Separate Process: ' . trim($log_file);
		}

		return $report;
	}
	public static function __post_test_run()
	{
		// Progressively upload result file at end of each test execution
		if(self::$progressive_result_uploads)
		{
			self::upload_progressive_test_result();
		}
	}
	private static function upload_progressive_test_result()
	{
		if(!self::$progressive_result_uploads || !(self::$test_run_manager->result_file instanceof pts_result_file))
		{
			return false;
		}

		$composite_xml = self::$test_run_manager->result_file->get_xml();
		return phoromatic::upload_to_remote_server(array(
			'r' => 'result_upload',
			'sched' => self::$p_schedule_id,
			'bid' => self::$benchmark_ticket_id,
			'o' => self::$p_save_identifier,
			'ts' => self::$p_trigger_id,
			'composite_xml' => base64_encode($composite_xml),
			'composite_xml_hash' => sha1($composite_xml),
			'progressive_upload' => 1
			));
	}
	private static function upload_test_result(&$result_file, $upload_system_logs = true, $schedule_id = 0, $save_identifier = null, $trigger = null, $elapsed_time = 0, $benchmark_ticket_id = null)
	{
		$system_logs = null;
		$system_logs_hash = null;
		if(self::$server_core_version < 10602)
		{
			// On newer PTS servers, upload as separate upload afterwards to better handle large log files that otherwise may go beyond max request size, etc

			// TODO: Potentially integrate this code below shared with pts_openbenchmarking_client into a unified function for validating system log files
			$system_log_dir = $result_file->get_system_log_dir();
			if(is_dir($system_log_dir) && ($upload_system_logs || pts_module::read_option('upload_system_logs', 0) != 0))
			{
				$is_valid_log = true;
				if(pts_client::$skip_log_file_type_checks == false)
				{
					// For security/malicious purposes, ensure system log directory only contains text files
					$is_valid_log = pts_file_io::directory_only_contains_text_files($system_log_dir);
				}

				if($is_valid_log)
				{
					$system_logs_zip = pts_client::create_temporary_file('.zip');
					pts_compression::zip_archive_create($system_logs_zip, $system_log_dir);

					if(filesize($system_logs_zip) == 0)
					{
						pts_client::$pts_logger && pts_client::$pts_logger->log('System log ZIP file failed to generate. Missing PHP ZIP support?');
					}
					else if(pts_client::$skip_log_file_type_checks == false && filesize($system_logs_zip) > 2097152)
					{
						// If it's over 2MB, probably too big
						pts_client::$pts_logger && pts_client::$pts_logger->log('System log ZIP file too big to upload');
					}
					else
					{
						$system_logs = base64_encode(file_get_contents($system_logs_zip));
						$system_logs_hash = sha1($system_logs);
					}
					unlink($system_logs_zip);
				}
			}
		}

		$composite_xml = $result_file->get_xml();
		$composite_xml_hash = sha1($composite_xml);
		$composite_xml_type = 'composite_xml';

		// Compress the result file XML if it's big
		if(isset($composite_xml[50000]) && function_exists('gzdeflate'))
		{
			$composite_xml_gz = gzdeflate($composite_xml);
			if($composite_xml_gz != false)
			{
				$composite_xml = $composite_xml_gz;
				$composite_xml_type = 'composite_xml_gz';
			}
		}

		// Upload to Phoromatic
		$times_tried = 0;
		do
		{
			if($times_tried > 0)
			{
				sleep(rand(5, 20));
			}

			$res = phoromatic::upload_to_remote_server(array(
				'r' => 'result_upload',
				//'ob' => $ob_data['id'],
				'sched' => $schedule_id,
				'bid' => $benchmark_ticket_id,
				'o' => $save_identifier,
				'ts' => $trigger,
				'et' => $elapsed_time,
				$composite_xml_type => base64_encode($composite_xml),
				'composite_xml_hash' => $composite_xml_hash,
				'system_logs_zip' => $system_logs,
				'system_logs_hash' => $system_logs_hash,
				'progressive_upload' => (self::$progressive_result_uploads ? 2 : 0)
				));

			$times_tried++;
		}
		while($res == false && $times_tried < 4);

		$server_response = json_decode($res, true);

		if(isset($server_response['phoromatic']['upload_id']) && !empty($server_response['phoromatic']['upload_id']))
		{
			// On newer PTS servers, upload as separate upload afterwards to better handle large log files that otherwise may go beyond max request size, etc

			$log_types = array();

			if($upload_system_logs)
			{
				$log_types['system-logs'] = $result_file->get_system_log_dir();
			}

			if(pts_module::read_option('upload_install_logs', 0) != 0)
			{
				$log_types['installation-logs'] = $result_file->get_test_installation_log_dir();
			}
			if(pts_module::read_option('upload_run_logs', 0) != 0)
			{
				$log_types['test-logs'] = $result_file->get_test_log_dir();
			}

			foreach($log_types as $index => $log_dir)
			{
				if(is_dir($log_dir))
				{
					$is_valid_log = true;
					if(pts_client::$skip_log_file_type_checks == false)
					{
						// For security/malicious purposes, ensure system log directory only contains text files
						$is_valid_log = pts_file_io::directory_only_contains_text_files($log_dir);
					}

					if($is_valid_log)
					{
						$system_logs_zip = pts_client::create_temporary_file('.zip');
						pts_compression::zip_archive_create($system_logs_zip, $log_dir);

						if(filesize($system_logs_zip) == 0)
						{
							pts_client::$pts_logger && pts_client::$pts_logger->log($index . ' log ZIP file failed to generate. Missing PHP ZIP support?');
						}
						else
						{
							$system_logs = base64_encode(file_get_contents($system_logs_zip));
							$system_logs_hash = sha1($system_logs);
						}
						unlink($system_logs_zip);

						phoromatic::upload_to_remote_server(array(
							'r' => 'result_log_upload',
							'i' => $server_response['phoromatic']['upload_id'],
							'system_logs_type' => $index,
							'system_logs_zip' => $system_logs,
							'system_logs_hash' => $system_logs_hash
							));
					}
				}
			}
		}

		return $res;
	}
	private static function upload_stress_log($stress_log)
	{
		// Upload Logs to Phoromatic
		if($stress_log == null || self::$benchmark_ticket_id == null)
		{
			return;
		}

		$times_tried = 0;
		do
		{
			if($times_tried > 0)
			{
				sleep(rand(5, 20));
			}

			$res = phoromatic::upload_to_remote_server(array(
				'r' => 'stress_log_upload',
				'bid' => self::$benchmark_ticket_id,
				'l' => pts_user_io::strip_ansi_escape_sequences($stress_log)
				));

			$times_tried++;
		}
		while($res == false && $times_tried < 4);

		return $res;
	}
	public static function upload_stress_log_sane($stress_log)
	{
		static $last_log_upload = 0;

		if(time() > ($last_log_upload + 60))
		{
			self::upload_stress_log($stress_log);
			$last_log_upload = time();
		}
	}
	public static function recent_phoromatic_server_results()
	{
		self::setup_server_addressing();
		$server_response = phoromatic::upload_to_remote_server(array('r' => 'list_results'), null, null, null, true);
		$server_response = json_decode($server_response, true);

		if(isset($server_response['phoromatic']['results']) && !empty($server_response['phoromatic']['results']))
		{
			foreach($server_response['phoromatic']['results'] as $pprid => $result)
			{
				echo pts_client::cli_just_bold($result['Title']) . ' ' . $pprid . ' ' . date('j M H:i', strtotime($result['UploadTime'])) . PHP_EOL;
				echo '   ' . $result['SystemName'] . ' ' . $result['GroupName'];
				echo PHP_EOL . PHP_EOL;
			}
		}
		else
			echo PHP_EOL . 'No Phoromatic Server results discovered.';

		echo PHP_EOL;
	}
	public static function clone_phoromatic_server_result($args)
	{
		self::setup_server_addressing();

		$id = $args[0];
		$server_response = phoromatic::upload_to_remote_server(array('r' => 'clone_result', 'i' => $id), null, null, null, true);
		$server_response = json_decode($server_response, true);

		if(isset($server_response['phoromatic']['result']['composite_xml']) && !empty($server_response['phoromatic']['result']['composite_xml']))
		{
			$composite_xml = base64_decode($server_response['phoromatic']['result']['composite_xml']);
			$result_file = new pts_result_file($composite_xml);
			// TODO XXX: Add system log downloading support
			pts_client::save_test_result($id . '/composite.xml', $result_file->get_xml(), true);
			echo PHP_EOL . 'Result File Saved As: ' . $id . PHP_EOL . PHP_EOL;
		}
		else
			echo PHP_EOL . 'No Phoromatic result found.' . PHP_EOL;
	}
	private static function run_client_update_script($update_script)
	{
		static $last_update_script_check_time = 0;

		// Don't keep checking it so check no more than every 30 minutes
		if($last_update_script_check_time < (time() - (60 * 30)) && !empty($update_script))
		{
			$last_update_script_check_time = time();
			$update_file = pts_client::create_temporary_file();
			$update_script = str_replace("\r", PHP_EOL, $update_script);
			file_put_contents($update_file, $update_script);
			phoromatic::update_system_status('Running Phoromatic Update Script');
			$env_vars = array();

			$append_cmd = '';
			if(phodevi::is_windows() && is_executable('C:\cygwin64\bin\bash.exe') && strpos($update_script, '#!/bin') !== false)
			{
				$cmd = 'C:\cygwin64\bin\bash.exe';
			}
			else if(phodevi::is_windows() && is_executable('C:\Windows\System32\cmd.exe'))
			{
				$cmd = 'C:\Windows\System32\cmd.exe /c';
			}
			else if(pts_client::executable_in_path('bash'))
			{
				$cmd = 'bash';
				$append_cmd = ' 2>&1';
			}
			else
			{
				$cmd = 'sh';
				$append_cmd = ' 2>&1';
			}

			pts_client::$pts_logger && pts_client::$pts_logger->log('Running Update Script: ' . $cmd . ' ' . $update_file . $append_cmd);
			$update_output = pts_client::shell_exec($cmd . ' ' . $update_file . $append_cmd, $env_vars);
			if(trim($update_output) != '')
			{
				pts_client::$pts_logger && pts_client::$pts_logger->log('Update Script Output: ' . trim($update_output));
			}
			unlink($update_file);
			phoromatic::update_system_status('Phoromatic Update Script Completed');
			self::tick_thread(true);
		}
	}
	private static function set_user_context($context_script, $trigger, $schedule_id, $process)
	{
		if(!empty($context_script))
		{
			$context_file = pts_client::create_temporary_file();
			file_put_contents($context_file, $context_script);
			chmod($context_file, 0755);

			pts_file_io::mkdir(pts_module::save_dir());
			$storage_path = pts_module::save_dir() . 'memory.pt2so';
			$storage_object = pts_storage_object::recover_from_file($storage_path);
			$notes_log_file = pts_module::save_dir() . sha1($trigger . $schedule_id . $process);

			// We check to see if the context was already set but the system rebooted or something in that script
			if($storage_object == false)
			{
				$storage_object = new pts_storage_object(true, true);
			}
			else if($storage_object->read_object('last_set_context_trigger') == $trigger && $storage_object->read_object('last_set_context_schedule') == $schedule_id && $storage_object->read_object('last_set_context_process') == $process)
			{
				// If the script already ran once for this trigger, don't run it again
				self::check_user_context_log($trigger, $schedule_id, $process, $notes_log_file, null);
				return false;
			}

			$storage_object->add_object('last_set_context_trigger', $trigger);
			$storage_object->add_object('last_set_context_schedule', $schedule_id);
			$storage_object->add_object('last_set_context_process', $process);
			$storage_object->save_to_file($storage_path);

			phoromatic::update_system_status('Setting context for: ' . $schedule_id . ' - ' . $trigger . ' - ' . $process);

			// Run the set context script
			$env_vars['PHOROMATIC_TRIGGER'] = $trigger;
			$env_vars['PHOROMATIC_SCHEDULE_ID'] = $schedule_id;
			$env_vars['PHOROMATIC_SCHEDULE_PROCESS'] = $process;
			$env_vars['PHOROMATIC_LOG_FILE'] = $notes_log_file;
			$log_output = pts_client::shell_exec($context_file . ' ' . $trigger . ' 2>&1', $env_vars);
			self::check_user_context_log($trigger, $schedule_id, $process, $notes_log_file, $log_output);

			// Just simply return true for now, perhaps check exit code status and do something
			return true;
		}

		return false;
	}
	private static function check_user_context_log($trigger, $schedule_id, $process, $log_file, $log_output)
	{
		if(is_file($log_file) && ($lf_conts = pts_file_io::file_get_contents($log_file)) != null)
		{
			$context_log = $lf_conts;
			unlink($log_file);
		}
		else
		{
			$context_log = trim($log_output);
		}

		if($context_log != null)
		{
			$server_response = phoromatic::upload_to_remote_server(array(
				'r' => 'user_context_log',
				'sched' => $schedule_id,
				'ts' => $trigger,
				'i' => $process,
				'o' => $context_log,
				));
		}
	}
	protected static function upload_test_install_manifest()
	{
		$json_to_upload = array();
		foreach(pts_tests::tests_installations_with_metadata() as $test_profile)
		{
			if($test_profile->test_installation)
			{
				$json_to_upload[$test_profile->get_identifier()] = $test_profile->test_installation->get_array();
			}
		}

		if(!empty($json_to_upload))
		{
			$server_response = phoromatic::upload_to_remote_server(array(
				'r' => 'test_install_manifest',
				'manifest' => json_encode($json_to_upload)
				));
		}
	}
	public static function __post_install_process()
	{
		self::upload_test_install_manifest();
	}
	public static function __post_run_process()
	{
		self::upload_test_install_manifest();
	}
	public static function __pre_test_install(&$test_install_request)
	{
	/*	if(self::$has_run_server_setup_func == false)
		{
			self::setup_server_addressing();
		}
	*/
	// XXX finish wiring in the above code to various parts for making auto-reporting from clients

		if(!self::$is_running_as_phoromatic_node)
		{
			return false;
		}

		static $last_update_time = 0;

		if(time() > ($last_update_time + 30))
		{
			phoromatic::update_system_status('Installing: ' . $test_install_request->test_profile->get_identifier());
			$last_update_time = time();
		}
	}
	public static function __pre_test_run($pts_test_result)
	{
		if(!self::$is_running_as_phoromatic_node)
		{
			return false;
		}

		if(self::$in_stress_mode)
		{
			static $time_in_stress_run = 0;

			$msg = 'Stress-Run Testing';
			if(($time_in_stress_run + (60 * 60)) > time())
			{
				// Don't report this same string so often...
				return;
			}
			$time_in_stress_run = time();
		}
		else
		{
			$msg = 'Running: ' . $pts_test_result->test_profile->get_identifier() . ($pts_test_result->get_arguments_description() != null ? ' [' . $pts_test_result->get_arguments_description() . ']' : null);
		}

		phoromatic::update_system_status($msg,
			ceil(self::$test_run_manager->get_estimated_run_time() / 60),
			self::$test_run_manager->get_percent_complete(),
			null,
			ceil($pts_test_result->get_estimated_run_time() / 60));
	}
	public static function __event_results_saved($test_run_manager)
	{
		/*if(pts_env::read('AUTO_UPLOAD_RESULTS_TO_PHOROMATIC') && pts_module::is_module_setup())
		{
			phoromatic::upload_unscheduled_test_results($test_run_manager->get_file_name());
		}*/
	}
	public static function __event_pre_run_error($error_obj)
	{
		list($test_profile, $error_msg) = $error_obj;

		$server_response = phoromatic::upload_to_remote_server(array(
			'r' => 'error_report',
			'sched' => self::$p_schedule_id,
			'ts' => self::$p_trigger_id,
			'err' => $error_msg,
			'ti' => $test_profile->get_identifier(),
			'o' => ''
			));
	}
	public static function __event_run_error($error_obj)
	{
		list($test_run_manager, $test_run_request, $error_msg) = $error_obj;

		if(stripos('Download Failed', $error_msg) !== false || stripos('check-sum of the downloaded file failed', $error_msg) !== false || stripos('attempting', $error_msg) !== false)
			return false;

		$server_response = phoromatic::upload_to_remote_server(array(
			'r' => 'error_report',
			'sched' => self::$p_schedule_id,
			'ts' => self::$p_trigger_id,
			'err' => $error_msg,
			'ti' => $test_run_request->test_profile->get_identifier(),
			'o' => $test_run_request->get_arguments_description()
			));
		//pts_client::$pts_logger && pts_client::$pts_logger->log('TEMP DEBUG MESSAGE: ' . $server_response);
	}
	public static function __event_reboot($passed_obj)
	{
		$reboot_msg = 'Attemtping system reboot';
		if($passed_obj != null)
		{
			if($passed_obj instanceof pts_test_result)
			{
				$reboot_msg = 'Attempting system reboot, requested by ' . $passed_obj->test_profile->get_identifier();
			}
			else if($passed_obj instanceof pts_test_profile)
			{
				$reboot_msg = 'Attempting system reboot, requested by ' . $passed_obj->get_identifier();
			}
		}
		phoromatic::update_system_status($reboot_msg);
	}
}

?>
