diff --git a/file-layout.txt b/file-layout.txt new file mode 100644 index 0000000..72e2583 --- /dev/null +++ b/file-layout.txt @@ -0,0 +1,61 @@ + +File Layout +=========== + + 2024-12-04 + + This is how the files are laid out. Important notes are the split between web and cli, db and rpc. + + Service files, or CLI + --------------------- + + Service Only + These files should not contain RPC. This is to remove the likelyhood of rpc blocking. Since the main app + is not able to respond to its own RPC + + ./svc/main.php + ./svc/db-reads.php + ./svc/db-writes.php + + Service Child Process Only + These files can have both database and RPC. This is because the child has been forked from main and is a legitimate + separate (child) process. It can invoke RPC to main comfortably. + + ./svc/backup.php + + Shared Files + ------------ + + These files contain no RPC and no DB. This allows them to be included in any process. They contain code of the most + general nature. + + ./shd/common.php (some old code needs to be removed) + ./lib/time.php + ./inc/paths.php (this will be upgraded and likely split into path-files specific to each section) + + Web-server Files + ---------------- + + These files specialise in handling the GUI, browser interaction using JS and HTML and session life-cycle. + No DB code allowed here. All calls but go via RPC to main.php. + + ./inc/bu_list_content.php + ./inc/secure.php + ./inc/pagemap.php + ./index.php + ./phpinfo.php + ./post_handler.php + ./inc/new-file-bu.php + ./inc/login.php + ./inc/edit-sh-bu.php + + Libraries + + (included as required) + ./lib/sysVcom.php + + + + + + diff --git a/html/bin/db-site-insert-all.php b/html/bin/db-site-insert-all.php index 4b16e73..24868ad 100644 --- a/html/bin/db-site-insert-all.php +++ b/html/bin/db-site-insert-all.php @@ -1,7 +1,6 @@ the name/path of this script //$id = $argv[1]; diff --git a/html/inc/bu-common.php b/html/inc/bu-common.php deleted file mode 100644 index 18dc25b..0000000 --- a/html/inc/bu-common.php +++ /dev/null @@ -1,258 +0,0 @@ -> $g_data_dir/$fn_e 1>> $g_data_dir/$fn_o"; - - $output = array(); - $result_code = null; - if( exec( $command, $output, $result_code )===FALSE){ - set_bu_running( $buid, false, $bu_test ); - return "rsync failed"; - } - set_bu_running( $buid, false, $bu_test ); - - // can we find any error strings in the error file? - $str_errs = file_get_contents( "$g_data_dir/$fn_e" ); - $pos = strpos($str_errs, 'error' ); - if($pos===FALSE){ - $pos = strpos($str_errs, 'failed' ); - } - if($pos===FALSE) return; - - //place it in the database - set_bu_error( $buid, $str_errs ); - -} - - -/** - * This function is now used directly by the backup service. - */ -function run_backup_DB( $bu_test, $buid, $bu_src, $bu_dest ){ - - global $g_data_dir; - - if(!set_bu_running( $buid, true, $bu_test )){ - return "Could not set the DB flag"; - } - - if(!is_dir($bu_dest) ){ - return "Destination ($bu_dest) is not a directory"; - } - - if($bu_test){ - $fn_o = getFilename_out( $buid, $bu_test, false ); - $bu_dest = "$g_data_dir"; - }else{ - $fn_o = "$bu_src.sql"; - set_bu_error( $buid, '' ); //clear any previous error - //set_last_run_date( $buid, gmdate("Y-m-d H:i:s") ); - } - - // create the error files with the ID in the first line - $fn_e = getFilename_out( $buid, $bu_test, true ); - exec("echo 'ID: $buid' > $g_data_dir/$fn_e"); - - // bash command - //mysqldump johntest > /home/johnp/tmp/dbs/johntest.sql - - $command = "mysqldump $bu_src 2>> $g_data_dir/$fn_e 1>> $bu_dest/$fn_o"; - - $output = array(); - $result_code = null; - if( exec( $command, $output, $result_code )===FALSE){ - set_bu_running( $buid, false, $bu_test ); - return "mysqldump failed"; - } - set_bu_running( $buid, false, $bu_test ); - - // can we find any error strings in the error file? - $str_errs = file_get_contents( "$g_data_dir/$fn_e" ); - $pos = strpos($str_errs, 'error' ); - if($pos===FALSE){ - $pos = strpos($str_errs, 'failed' ); - } - if($pos===FALSE) return; - - //place it in the database - set_bu_error( $buid, $str_errs ); - -} - -function getFilename_out( $buid, bool $bu_test, bool $isErr=false ){ - return getFilename_tmp( $buid, $bu_test?'t':'n', $isErr? 'err': 'op' ); -} - -/** - * Create an output file for the backup (forked) process to use. These are essentually - * temporary files which may be deleted after the back process completes. - * - * buid: backup record id - * - * fn_type: - * t: for test files - * n: for non-test files - * x: for file exclusion list - * - * fn_stream: - * op: output stream - * err: error stream - */ -function getFilename_tmp( $buid, $fn_type='t', $fn_stream='op' ){ - return "bu-$fn_type-$fn_stream-$buid.txt"; -} - -function set_last_run_date( $buid, $str_dt ){ - global $g_database_path; - $db = new SQLite3( __DIR__."/$g_database_path" ); - $p = $db->prepare("update BULIST set LastRunDt=:str_dt where BUID=:buid"); - $p->bindValue(':str_dt', $str_dt ); - $p->bindValue(':buid', $buid ); - $p->execute(); - - if( $db->lastErrorCode()!==0){ - return $db->lastErrorCode(); - } -} - -function set_sched_date( $buid, $dt_new ){ - - global $g_database_path; - - $db = new SQLite3( __DIR__."/$g_database_path" ); - - $p = $db->prepare("update BULIST set BU_DATE=:str_dt where BUID=:buid"); - $p->bindValue(':str_dt', $dt_new ); - $p->bindValue(':buid', $buid ); - $p->execute(); - - if( $db->lastErrorCode()!==0){ - return $db->lastErrorCode(); - } -} - -?> diff --git a/html/inc/bu_list_content.php b/html/inc/bu_list_content.php index f3c23c1..82002a6 100644 --- a/html/inc/bu_list_content.php +++ b/html/inc/bu_list_content.php @@ -1,14 +1,5 @@ getTraceAsString(); - } -} - -/** - * e.g. split - * ../db/stanhopetest/img/img_52548 - * into - * [ ../db/stanhopetest/img/ , img_52548 ] - * - */ -function split_file_path( $ffn ){ - $fn = basename( $ffn ); // img_52548 - //$fp = substr( $ffn, 0, strlen($ffn) - strlen($fn) ); - $fp = dirname( $ffn ) . '/'; - return [ $fp, $fn ]; -} - -/** - * Returns true if it is possible to create this directory on the file system. - * Includes the possibility of multiple directories. - */ -function is_creatable_dir( $dirs ){ - $dir = $dirs; - while(!file_exists($dir)){ - $dir = dirname( $dir ); - } - return is_writable($dir); -} - -/** - * Function to make directories and actually SET the permissions - * https://www.php.net/manual/en/function.chmod.php - * - * $perms is an octal number - * e.g. mkdirs( $dir, 02770 ); //770 and g+s - */ -function mkdirs( $dirs, $perms ){ - $dir = $dirs; - while(!file_exists($dir)){ - $dir = dirname( $dir ); - } - $rootDir = $dir; - if(!file_exists($dirs)) mkdir( $dirs, 0750, true ); //perms not always set - $dir = $dirs; - while( $dir!=$rootDir) { - chmod( $dir, $perms ); - $dir = dirname( $dir ); - } -} - -/** - * Return the contents of a url as a string using the curl functions. - */ -function getContentsFromUrl( $url ){ - - $ch = curl_init( $url ); - //$fp = fopen("example_homepage.txt", "w"); - $fp = tmpfile(); - - curl_setopt($ch, CURLOPT_FILE, $fp); - curl_setopt($ch, CURLOPT_HEADER, 0); - - curl_exec($ch); - $c_err = curl_error($ch); - curl_close($ch); - if($c_err) { - error_log( $c_err ); - fclose($fp); - return; - } - - $f_path = stream_get_meta_data($fp)['uri']; - $rtn = file_get_contents( $f_path ); - fclose($fp); - - return $rtn; -} - -/** - * Set the given key-value pair. - * - * Returns the record ID. - */ -function params_set( $key, $val ) { - - $rec = R::findOne( 'params', 'pkey=?', [$key] ); - if($rec) { - $rec->pval = $val; - R::store( $rec ); - return $rec->id; - } - - $rec = R::dispense( 'params' ); - $rec->pkey = $key; - $rec->pval = $val; - return R::store( $rec ); -} - -function params_get_date( $key ){ - return strtotime( params_get($key) ); -} - -/** - * Returns the value for the given key. Returns null if no such key exists. - */ -function params_get( $key ){ - - $rec = R::findOne( 'params', 'pkey=?', [$key] ); - if(!$rec) return null; - return $rec->pval; -} - -/** - * A novel way of creating a unique ID each time a call is made. The new ID is - * returned. - * - * e.g. - * This can be used to give a client which might want to create temporary unique IDs. It is - * called once per client refresh and used to ensure uniqueness of certain CSS data points. - */ -function params_unique( ) { - - $rec = R::dispense( 'params' ); - $rec->pkey = 'unique'; - $id = R::store( $rec ); - R::exec( "delete from params where id<$id and pkey='unique'" ); - return $id; -} - -/** - * Return a string representation for the given dom node suitable for debugging. - */ -function getNodeString( $rmElem ){ - - if($rmElem==null) return "Element is null"; - $str=""; - if( $rmElem->nodeType==XML_ELEMENT_NODE){ - $str .= $rmElem->nodeName; - $a = $rmElem->getAttribute("shb_compid"); - if($a){ - $str .= " shb_compid=$a"; - } - }else{ - $str .= 'type = '.$rmElem->nodeType; - } - return $str; -} - -/** - * Simple logger to write to a file. Using php error_log distorts all crlf pairs. This - * is a pain when you're trying to preserve them in html. - */ -function mylog( $msg ){ - - global $logging_on; - - if(!$logging_on) return; - $p = __DIR__ . "/../../shb.log"; - $h = fopen( $p, "a"); - if($h===false) return; - fwrite( $h, "$msg\n"); - fclose($h); - -} - -function generateNonce( $digitCount = 12 ){ - // e.g. 422466a05f24b09c978fa1f6 - return bin2hex( random_bytes($digitCount) ); -} - -/** - * Generate a randon username starting with a letter and containing only letters and numbers. - */ -function generateRndUserName( $digitCount = 10 ){ - $l1 = generateRndStrUsing( 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz', 1 ); - return $l1 . generateRndStrUsing( BASE64_DIGITS, $digitCount-1 ); -} - -/** - * Generate a randon password with letters and numbers and some funny stuff. - */ -function generateRndPwd( $digitCount = 10 ){ - return generateRndStrUsing( '0123456789_!"£$%^&*()_-|\?/<>#~ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz', $digitCount ); -} - -/** - * Generate a randon string of asc char. Default consisting of only [0-9][a-z][A-Z] - */ -function generateRndStrUsing( $using=BASE64_DIGITS, $digitCount = 8 ){ - - $str = ''; - $maxlen = strlen($using); - for($i=0; $i<$digitCount; $i++){ - $iv = random_int(0, $maxlen-1); - $str .= mb_substr( $using, $iv, 1); //use mb_substr because £ is not and asc char! - } - return $str; -} - - -/** - * Remove all html from the given string returning just the text content. - */ -function removeHtml( $str ){ - //txt = txt.replace(/
/gi, '\n'); - return preg_replace( '/<\/?[^>]+>/i', '', $str ); -} - -function getHtmlOkAsString( $data ){ - $data = json_encode( $data ); - return "[\"OK\", $data ]"; -} - -function sendHtmlOk( $data="" ){ - echo getHtmlOkAsString( $data ); -} - -/** - * An effort to rectify my previously ill-thought out return encoding. You do not - * need to encode the data before calling this method. Any arrays will be json encoded. - */ -function sendHtmlOk_WithData( $data ){ - $data = json_encode( $data ); - echo "[\"OK\", $data ]"; -} - -function getHtmlErrorAsString( $strErrDesc = "--blank--" ){ - $data = json_encode( $strErrDesc ); - return "[\"Error\", $data]"; -} - -function sendHtmlError( $strErrDesc = "--blank--" ){ - $rtn = getHtmlErrorAsString( $strErrDesc ); - mylog( $rtn ); - echo $rtn; -} - -/** - * - */ -function getAttForPath( $name, $prefPath, $path){ - return "$name = \"$prefPath/$path\""; -} - -global $g_pub_path; - -/** - * If the browser=true, the file returned will be correct for adding to an href or src attribute so that the - * browser will find the link. - */ -function getDirForType( $filetype='html', $browser=false, $userid=-1 ){ - - global $g_pub_path; - - if($g_pub_path==null){ - $g_pub_path = params_get('pub_path'); - if($g_pub_path==null){ - throw new Exception("Publishing now requires a valid key('pub_path') in the params table for the publishing output path"); - } - - if(str_ends_with( $g_pub_path, '/') ) $g_pub_path = substr($g_pub_path, 0, strlen($g_pub_path)-1 ); - - //ensure all the directories exist - createUserDir( 'img', $userid ); - createUserDir( 'css', $userid ); - createUserDir( 'html', $userid ); - createUserDir( 'js', $userid ); - - } - - return getDirForType_( $filetype, $browser, $userid ); - -} - -function createUserDir( $dName, $userid ){ - - if(!file_exists(getDirForType_( $dName, false, $userid ))) { - $dir = getDirForType_( $dName, false, $userid ); - error_log("common.php: createUserDir: $dir"); - mkdir( $dir, 0770, true); - } -} - -/** - * Part of the new path handling system is to sepatate out path names from real files -*/ -function mkdirIfNotExists( $d ){ - if(!file_exists( $d ) ) { - error_log("common.php: createUserDir: $d"); - mkdir( $d, 0770, true ); - } -} - -/** - * For the moment, call getDirForType and not this function. In time, the setup routine will make the directories - * and eliminate the need for the check during a user session. - * - * If the browser flag is set, we assuming a document root sub-dir. But in time, this will need to also cater for - * a site serving the g_pub_path as root (wrt the browser). - * - */ -function getDirForType_( $filetype, $browser, $userid ){ - - global $g_pub_path; - $strPath = $g_pub_path; - - - if($userid>0){ - $strPath .= "_dbg_$userid"; - } - - switch($filetype){ - - case 'img': - if($browser) return "/$strPath/img/"; - return fxdPth( "$strPath/img/" ); - - case 'js': - if($browser) return "/$strPath/js/"; - return fxdPth( "$strPath/js/" ); - - case 'css': - if($browser) return "/$strPath/css/"; - return fxdPth( "$strPath/css/" ); - - case 'html': - default: - if($browser) return "/$strPath/"; - return fxdPth( "$strPath/" ); - - } -} - - -//NO white-space diff --git a/html/inc/delete-file-bu.php b/html/inc/delete-file-bu.php deleted file mode 100644 index 95b0fd6..0000000 --- a/html/inc/delete-file-bu.php +++ /dev/null @@ -1,18 +0,0 @@ -prepare('delete from BULIST where BUID=:buid'); - $db_st->bindValue(':buid', $id ); - $db_st->execute(); - - if( $db->lastErrorCode()!==0){ - return $db->lastErrorCode(); - } - -} - -?> diff --git a/html/inc/edit-sh-bu.php b/html/inc/edit-sh-bu.php index f737959..c297476 100644 --- a/html/inc/edit-sh-bu.php +++ b/html/inc/edit-sh-bu.php @@ -1,10 +1,5 @@ diff --git a/html/inc/pagemap.php b/html/inc/pagemap.php index e77ab29..2155037 100644 --- a/html/inc/pagemap.php +++ b/html/inc/pagemap.php @@ -5,7 +5,8 @@ */ -include_once __DIR__ . "/common.php"; +include_once __DIR__ . "/../shd/common.php"; + /** * The commands should all return true to indicate that they are valid calls from the client. @@ -14,8 +15,8 @@ if( $cmd=='backup-list-state' ){ - include_once __DIR__ . "/bu_list_content.php"; - sendHtmlOk_WithData( [ 'state', getbu_list_state() ] ); + //include_once __DIR__ . "/bu_list_content.php"; + sendHtmlOk_WithData( [ 'state', rpc( 'f:getbu_list_state' ) ] ); return true; } @@ -93,8 +94,12 @@ } if( $cmd=='edit_shed_bu' ){ - include_once __DIR__ . "/edit-sh-bu.php"; - $db_err = save_bu_schedule( $postvars ); + + //include_once __DIR__ . "/edit-sh-bu.php"; + //$db_err = save_bu_schedule( $postvars ); + + [$db_err] = rpc( 'f:save_bu_schedule', $postvars ); + if($db_err>0){ sendHtmlError( "DB error: $db_err" ); return true; @@ -111,8 +116,11 @@ if( $cmd=='new-item-bu' ){ - include_once __DIR__ . "/new-file-bu.php"; - $db_err = save_bu_item( $postvars ); + //include_once __DIR__ . "/new-file-bu.php"; + //$db_err = save_bu_item( $postvars ); + + [$db_err] = rpc( 'f:save_bu_item', $postvars ); + if($db_err>0){ sendHtmlError( "DB error: $db_err" ); return true; @@ -129,8 +137,11 @@ if( $cmd=='edit-item-bu' ){ - include_once __DIR__ . "/new-file-bu.php"; - $db_err = save_bu_item( $postvars, $postvars['id'] ); + //include_once __DIR__ . "/new-file-bu.php"; + //$db_err = save_bu_item( $postvars, $postvars['id'] ); + + [$db_err] = rpc( 'f:save_bu_item', $postvars, $postvars['id'] ); + if($db_err>0){ sendHtmlError( "DB error: $db_err" ); return true; @@ -146,9 +157,9 @@ } if( $cmd=='delete-bu' ){ - - include_once __DIR__ . "/delete-file-bu.php"; - $db_err = delete_bu_item( $postvars['id'] ); + + [$db_err] = rpc( 'f:delete_bu_item', $postvars['id'] ); + if($db_err>0){ sendHtmlError( "DB error: $db_err" ); return true; diff --git a/html/inc/secure.php b/html/inc/secure.php index 0e1450a..9995011 100644 --- a/html/inc/secure.php +++ b/html/inc/secure.php @@ -38,7 +38,7 @@ exit; } - include_once "common.php"; + include_once "../shd/common.php"; include_once "login.php"; if(authenticate( $_POST['username'], $_POST['password'], $_SESSION['svr_nonce'] ) ){ @@ -65,7 +65,7 @@ // likely that the page was just refreshed // --- Send back the login page --- -include_once "common.php"; +include_once "../shd/common.php"; $_SESSION['svr_nonce'] = generateNonce(); include_once "login.php"; diff --git a/html/inc/service_calls.php b/html/inc/service_calls.php deleted file mode 100644 index 792f726..0000000 --- a/html/inc/service_calls.php +++ /dev/null @@ -1,16 +0,0 @@ - diff --git a/html/js/main.js b/html/js/main.js index da9b765..5e0a3d7 100644 --- a/html/js/main.js +++ b/html/js/main.js @@ -170,10 +170,6 @@ let b_is_db_type = false; switch(instr){ - - case 'btn_test_1': - exec_instruction( instr, map_params ); - return; case 'btn_login': if( !valid_btn_login() ) return; @@ -302,7 +298,15 @@ switch(instr){ case 'btn_run_bu': - if(data[1][0]=='done'){ + if(data[1][0]!='ok:'){ + //latest msg contains similar format as used by the server rpc layer + console.log("odd message: " + data[1][0] ); + console.log( data ); + return; + } + bu_status = data[1][1]; + + if(bu_status=='done'){ // this comes from the test run, although I should really explicitly send back the 'test' flag let elem = document.getElementById( "ta_testoutput" ); if(elem==null){ @@ -313,8 +317,8 @@ return; } - if(data[1][0]!='running'){ - console.log("odd message: " + data[1][0] ); + if(bu_status!='running'){ + console.log("unknown message: " + bu_status ); console.log( data ); return; } diff --git a/html/lib/sysVcom.php b/html/lib/sysVcom.php index 8e6fd9b..125e603 100644 --- a/html/lib/sysVcom.php +++ b/html/lib/sysVcom.php @@ -28,6 +28,7 @@ $g_max_rcv = $g_max_chan+100; $g_max_data = $g_max_chan - 2; +$main_pid = 0; function rpc_setGlobals( $name, $value ){ @@ -42,6 +43,8 @@ $g_procName = $value; $g_k_caller = ftok( $g_procName, 'A'); //caller $g_k_reply = ftok( $g_procName, 'B'); //reply + debug_print( "g_k_caller key = $g_k_caller\n" ); + debug_print( "g_k_reply key = $g_k_reply\n" ); break; case 'max_chan': @@ -52,6 +55,11 @@ $g_max_rcv = $g_max_chan+100; $g_max_data = $g_max_chan - 2; //we use the first 2 bytes for msg chunks break; + + case 'main_pid': + global $main_pid; + $main_pid = (int)$value; + break; } } @@ -60,17 +68,44 @@ // The following functions all relate to making an RPC. // They use the php serialize() to send objects across the process boundary +/** + * Sends a call to the remote process. The call blocks until a reply is recieved. There is no unblocking version of this method. + * + * The underlying layer uses the process ID of this process and so it is not possible to send more than one message at a time + * because it would not be possible to correlate the replies. This is not a really problem because of the lack of multi-threading + * support in PHP. + */ function rpc( $funcName, ...$params ){ array_unshift( $params, $funcName); return unserialize( IPC( serialize( $params ) ) ); } +/** + * Returns the next message waiting on the queue if there is one. The value of $caller_id must be checked + * to ascertain the validity of the return value. + * + * A greater of zero means that either no message was received in + * the case of non-blocking or that a message has only partially been retrieved and is not yet ready. A positive value + * means that the message is ready and unserialize. + * + * $msgArr is of little use here because it contains the unserialized msg. It must be supplied though as it is + * used by the underlying layer as a storage of partially retrieved messages. + * + */ function rpc_listen( &$caller_id, &$msgArr, $bBlock = true ){ + //debug_print( "-- rpc_listen in\n" ); $caller_id = IPC_listen( $msgArr, $bBlock ); + //debug_print( "-- rpc_listen out\n" ); if($caller_id==0 && !$bBlock) return null; - return unserialize( $msgArr[$caller_id]['msg'] ); + $obj = unserialize( $msgArr[$caller_id]['msg'] ); + //done with the global message + unset( $msgArr[$caller_id] ); + return $obj; } +/** + * A server process listening for messages must use this method to reply. The sender will be waiting! + */ function rpc_reply( $caller_id, $obj ){ IPC_reply( $caller_id, serialize($obj) ); } @@ -78,18 +113,29 @@ //************************************************************************************************************************ // -// All functions fast this point relate to sending and recieving of a single message as a string without the concept +// All functions below relate to sending and recieving of a single message string without the concept // of a function or parameters. // function clear_IPC( $g_procName ){ - $q_caller = msg_get_queue( ftok( $g_procName, 'A') ); - msg_remove_queue($q_caller); + global $g_max_rcv; + $received_message_type=null; + $message=null; + $q_caller = msg_get_queue( ftok( $g_procName, 'A') ); + while( FALSE !== msg_receive( $q_caller, 0, $received_message_type, $g_max_rcv, $message, false, MSG_IPC_NOWAIT ) ){ + print("Dumping msg: $received_message_type\n"); + } + msg_remove_queue($q_caller); + $q_reply = msg_get_queue( ftok( $g_procName, 'B') ); + while( FALSE !== msg_receive( $q_reply, 0, $received_message_type, $g_max_rcv, $message, false, MSG_IPC_NOWAIT ) ){ + print("Dumping msg: $received_message_type\n"); + } msg_remove_queue($q_reply); + print("Queues cleared\n"); } /** @@ -106,11 +152,22 @@ global $g_k_reply; global $g_max_data; global $g_max_rcv; - + global $main_pid; + $q_caller = msg_get_queue($g_k_caller); $q_reply = msg_get_queue($g_k_reply); $pid = getmypid(); + if($main_pid==$pid){ + /* This is a safeguard so that you do not spend years trying to find out why a receive is blocked. + * The main process sets its pid (main_pid) at startup. Although it is conceivable that main.php might want to make an rpc call, it + * will not be a call to itself, rather another process. But this would need to happen on another queue but this API is only set up + * to use the 2 hard-coded queues currently. It is likely that you are trying to call a DB-function through rpc when main.php should + * be calling the db-function directly. + */ + throw new Exception("IPC calls are not allowed within the same process ($pid)"); + } + IPC_send( $q_caller, $pid, $str_data ); $msgArr = null; @@ -166,13 +223,20 @@ if(!$bBlock){ $flags = MSG_IPC_NOWAIT; } - if( FALSE === msg_receive( $queue, $rqd_msg_id, $received_message_type, $g_max_rcv, $message, true, $flags, $error_code ) ){ + //debug_print( "IPC_rcv in for id = $rqd_msg_id\n" ); + if( FALSE === msg_receive( $queue, $rqd_msg_id, $received_message_type, $g_max_rcv, $message, false, $flags, $error_code ) ){ + if(!$bBlock && $error_code==MSG_ENOMSG){ + //debug_print( "IPC_rcv out 1\n" ); return 0; } + //debug_print( "IPC_rcv out 2\n" ); + //I'm guessing at these error codes... //code 7 when g_max_rcv is too small + //code 43 when pipe was broken by server throw new Exception("Failed to rcv message err = $error_code"); } + //debug_print( "IPC_rcv out 3\n" ); //echo "Got msg part " . substr( $message, 0, 20 ) . "\n\n"; @@ -207,7 +271,7 @@ function IPC_send( $queue, $caller_id, $str_data ){ //print_r( msg_stat_queue($queue) ); - + global $procName; global $g_k_caller; global $g_k_reply; @@ -234,9 +298,10 @@ } //echo "send($caller_id) len = ".strlen($d_meta.$str_chunk)."\n"; + //debug_print( "IPC_send 2\n" ); $error_code = 0; - if( FALSE === msg_send( $queue, $caller_id, $d_meta.$str_chunk, true, true, $error_code ) ){ + if( FALSE === msg_send( $queue, $caller_id, $d_meta.$str_chunk, false, true, $error_code ) ){ //code 22 when the message is too large throw new Exception("Failed to send message err = $error_code"); } diff --git a/html/post_handler.php b/html/post_handler.php index 22d0bf2..2414ede 100644 --- a/html/post_handler.php +++ b/html/post_handler.php @@ -8,12 +8,16 @@ include_once __DIR__ . "/inc/paths.php"; +function debug_print( $msg ){ + error_log( $msg ); +} + include __DIR__."/lib/sysVcom.php"; rpc_setGlobals( 'procName', 'svc/BU-commander' ); rpc_setGlobals( 'max_chan', 8182 ); include_once __DIR__ . "/inc/secure.php"; -include_once __DIR__ . "/inc/common.php"; +include_once __DIR__ . "/shd/common.php"; if( !isset($_POST['action']) ){ error_log("Post Error: Missing value for 'action'"); diff --git a/html/shd/common.php b/html/shd/common.php new file mode 100644 index 0000000..240ad08 --- /dev/null +++ b/html/shd/common.php @@ -0,0 +1,352 @@ +getTraceAsString(); + } +} + +/** + * e.g. split + * ../db/stanhopetest/img/img_52548 + * into + * [ ../db/stanhopetest/img/ , img_52548 ] + * + */ +function split_file_path( $ffn ){ + $fn = basename( $ffn ); // img_52548 + //$fp = substr( $ffn, 0, strlen($ffn) - strlen($fn) ); + $fp = dirname( $ffn ) . '/'; + return [ $fp, $fn ]; +} + +/** + * Returns true if it is possible to create this directory on the file system. + * Includes the possibility of multiple directories. + */ +function is_creatable_dir( $dirs ){ + $dir = $dirs; + while(!file_exists($dir)){ + $dir = dirname( $dir ); + } + return is_writable($dir); +} + +/** + * Function to make directories and actually SET the permissions + * https://www.php.net/manual/en/function.chmod.php + * + * $perms is an octal number + * e.g. mkdirs( $dir, 02770 ); //770 and g+s + */ +function mkdirs( $dirs, $perms ){ + $dir = $dirs; + while(!file_exists($dir)){ + $dir = dirname( $dir ); + } + $rootDir = $dir; + if(!file_exists($dirs)) mkdir( $dirs, 0750, true ); //perms not always set + $dir = $dirs; + while( $dir!=$rootDir) { + chmod( $dir, $perms ); + $dir = dirname( $dir ); + } +} + +/** + * Return the contents of a url as a string using the curl functions. + */ +function getContentsFromUrl( $url ){ + + $ch = curl_init( $url ); + //$fp = fopen("example_homepage.txt", "w"); + $fp = tmpfile(); + + curl_setopt($ch, CURLOPT_FILE, $fp); + curl_setopt($ch, CURLOPT_HEADER, 0); + + curl_exec($ch); + $c_err = curl_error($ch); + curl_close($ch); + if($c_err) { + error_log( $c_err ); + fclose($fp); + return; + } + + $f_path = stream_get_meta_data($fp)['uri']; + $rtn = file_get_contents( $f_path ); + fclose($fp); + + return $rtn; +} + +/** + * Set the given key-value pair. + * + * Returns the record ID. + */ +function params_set( $key, $val ) { + + $rec = R::findOne( 'params', 'pkey=?', [$key] ); + if($rec) { + $rec->pval = $val; + R::store( $rec ); + return $rec->id; + } + + $rec = R::dispense( 'params' ); + $rec->pkey = $key; + $rec->pval = $val; + return R::store( $rec ); +} + +function params_get_date( $key ){ + return strtotime( params_get($key) ); +} + +/** + * Returns the value for the given key. Returns null if no such key exists. + */ +function params_get( $key ){ + + $rec = R::findOne( 'params', 'pkey=?', [$key] ); + if(!$rec) return null; + return $rec->pval; +} + +/** + * A novel way of creating a unique ID each time a call is made. The new ID is + * returned. + * + * e.g. + * This can be used to give a client which might want to create temporary unique IDs. It is + * called once per client refresh and used to ensure uniqueness of certain CSS data points. + */ +function params_unique( ) { + + $rec = R::dispense( 'params' ); + $rec->pkey = 'unique'; + $id = R::store( $rec ); + R::exec( "delete from params where id<$id and pkey='unique'" ); + return $id; +} + +/** + * Return a string representation for the given dom node suitable for debugging. + */ +function getNodeString( $rmElem ){ + + if($rmElem==null) return "Element is null"; + $str=""; + if( $rmElem->nodeType==XML_ELEMENT_NODE){ + $str .= $rmElem->nodeName; + $a = $rmElem->getAttribute("shb_compid"); + if($a){ + $str .= " shb_compid=$a"; + } + }else{ + $str .= 'type = '.$rmElem->nodeType; + } + return $str; +} + +/** + * Simple logger to write to a file. Using php error_log distorts all crlf pairs. This + * is a pain when you're trying to preserve them in html. + */ +function mylog( $msg ){ + + global $logging_on; + + if(!$logging_on) return; + $p = __DIR__ . "/../../shb.log"; + $h = fopen( $p, "a"); + if($h===false) return; + fwrite( $h, "$msg\n"); + fclose($h); + +} + +function generateNonce( $digitCount = 12 ){ + // e.g. 422466a05f24b09c978fa1f6 + return bin2hex( random_bytes($digitCount) ); +} + +/** + * Generate a randon username starting with a letter and containing only letters and numbers. + */ +function generateRndUserName( $digitCount = 10 ){ + $l1 = generateRndStrUsing( 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz', 1 ); + return $l1 . generateRndStrUsing( BASE64_DIGITS, $digitCount-1 ); +} + +/** + * Generate a randon password with letters and numbers and some funny stuff. + */ +function generateRndPwd( $digitCount = 10 ){ + return generateRndStrUsing( '0123456789_!"£$%^&*()_-|\?/<>#~ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz', $digitCount ); +} + +/** + * Generate a randon string of asc char. Default consisting of only [0-9][a-z][A-Z] + */ +function generateRndStrUsing( $using=BASE64_DIGITS, $digitCount = 8 ){ + + $str = ''; + $maxlen = strlen($using); + for($i=0; $i<$digitCount; $i++){ + $iv = random_int(0, $maxlen-1); + $str .= mb_substr( $using, $iv, 1); //use mb_substr because £ is not and asc char! + } + return $str; +} + + +/** + * Remove all html from the given string returning just the text content. + */ +function removeHtml( $str ){ + //txt = txt.replace(/
/gi, '\n'); + return preg_replace( '/<\/?[^>]+>/i', '', $str ); +} + +function getHtmlOkAsString( $data ){ + $data = json_encode( $data ); + return "[\"OK\", $data ]"; +} + +function sendHtmlOk( $data="" ){ + echo getHtmlOkAsString( $data ); +} + +/** + * An effort to rectify my previously ill-thought out return encoding. You do not + * need to encode the data before calling this method. Any arrays will be json encoded. + */ +function sendHtmlOk_WithData( $data ){ + $data = json_encode( $data ); + echo "[\"OK\", $data ]"; +} + +function getHtmlErrorAsString( $strErrDesc = "--blank--" ){ + $data = json_encode( $strErrDesc ); + return "[\"Error\", $data]"; +} + +function sendHtmlError( $strErrDesc = "--blank--" ){ + $rtn = getHtmlErrorAsString( $strErrDesc ); + mylog( $rtn ); + echo $rtn; +} + +/** + * + */ +function getAttForPath( $name, $prefPath, $path){ + return "$name = \"$prefPath/$path\""; +} + +global $g_pub_path; + +/** + * If the browser=true, the file returned will be correct for adding to an href or src attribute so that the + * browser will find the link. + */ +function getDirForType( $filetype='html', $browser=false, $userid=-1 ){ + + global $g_pub_path; + + if($g_pub_path==null){ + $g_pub_path = params_get('pub_path'); + if($g_pub_path==null){ + throw new Exception("Publishing now requires a valid key('pub_path') in the params table for the publishing output path"); + } + + if(str_ends_with( $g_pub_path, '/') ) $g_pub_path = substr($g_pub_path, 0, strlen($g_pub_path)-1 ); + + //ensure all the directories exist + createUserDir( 'img', $userid ); + createUserDir( 'css', $userid ); + createUserDir( 'html', $userid ); + createUserDir( 'js', $userid ); + + } + + return getDirForType_( $filetype, $browser, $userid ); + +} + +function createUserDir( $dName, $userid ){ + + if(!file_exists(getDirForType_( $dName, false, $userid ))) { + $dir = getDirForType_( $dName, false, $userid ); + error_log("common.php: createUserDir: $dir"); + mkdir( $dir, 0770, true); + } +} + +/** + * Part of the new path handling system is to sepatate out path names from real files +*/ +function mkdirIfNotExists( $d ){ + if(!file_exists( $d ) ) { + error_log("common.php: createUserDir: $d"); + mkdir( $d, 0770, true ); + } +} + +/** + * For the moment, call getDirForType and not this function. In time, the setup routine will make the directories + * and eliminate the need for the check during a user session. + * + * If the browser flag is set, we assuming a document root sub-dir. But in time, this will need to also cater for + * a site serving the g_pub_path as root (wrt the browser). + * + */ +function getDirForType_( $filetype, $browser, $userid ){ + + global $g_pub_path; + $strPath = $g_pub_path; + + + if($userid>0){ + $strPath .= "_dbg_$userid"; + } + + switch($filetype){ + + case 'img': + if($browser) return "/$strPath/img/"; + return fxdPth( "$strPath/img/" ); + + case 'js': + if($browser) return "/$strPath/js/"; + return fxdPth( "$strPath/js/" ); + + case 'css': + if($browser) return "/$strPath/css/"; + return fxdPth( "$strPath/css/" ); + + case 'html': + default: + if($browser) return "/$strPath/"; + return fxdPth( "$strPath/" ); + + } +} + + +//NO white-space diff --git a/html/svc/backup.php b/html/svc/backup.php new file mode 100644 index 0000000..4049302 --- /dev/null +++ b/html/svc/backup.php @@ -0,0 +1,194 @@ +> $g_data_dir/$fn_e 1>> $g_data_dir/$fn_o"; + + $output = array(); + $result_code = null; + if( exec( $command, $output, $result_code )===FALSE){ + set_bu_running( $buid, false, $bu_test ); + return "rsync failed"; + } + set_bu_running( $buid, false, $bu_test ); + + // can we find any error strings in the error file? + $str_errs = file_get_contents( "$g_data_dir/$fn_e" ); + $pos = strpos($str_errs, 'error' ); + if($pos===FALSE){ + $pos = strpos($str_errs, 'failed' ); + } + if($pos===FALSE) return; + + //place it in the database + rpc( 'f:set_bu_error', $buid, $str_errs ); +} + + +/** + * This function is now used directly by the backup service. + */ +function run_backup_DB( $bu_test, $buid, $bu_src, $bu_dest ){ + + global $g_data_dir; + + if(!set_bu_running( $buid, true, $bu_test )){ + return "Could not set the DB flag"; + } + + if(!is_dir($bu_dest) ){ + return "Destination ($bu_dest) is not a directory"; + } + + if($bu_test){ + $fn_o = getFilename_out( $buid, $bu_test, false ); + $bu_dest = "$g_data_dir"; + }else{ + $fn_o = "$bu_src.sql"; + rpc( 'f:set_bu_error', $buid, '' ); //clear any previous error + rpc( 'f:set_last_run_date', $buid, gmdate("Y-m-d H:i:s") ); + } + + // create the error files with the ID in the first line + $fn_e = getFilename_out( $buid, $bu_test, true ); + exec("echo 'ID: $buid' > $g_data_dir/$fn_e"); + + // bash command + //mysqldump johntest > /home/johnp/tmp/dbs/johntest.sql + + $command = "mysqldump $bu_src 2>> $g_data_dir/$fn_e 1>> $bu_dest/$fn_o"; + + $output = array(); + $result_code = null; + if( exec( $command, $output, $result_code )===FALSE){ + set_bu_running( $buid, false, $bu_test ); + return "mysqldump failed"; + } + set_bu_running( $buid, false, $bu_test ); + + // can we find any error strings in the error file? + $str_errs = file_get_contents( "$g_data_dir/$fn_e" ); + $pos = strpos($str_errs, 'error' ); + if($pos===FALSE){ + $pos = strpos($str_errs, 'failed' ); + } + if($pos===FALSE) return; + + //place it in the database + rpc( 'f:set_bu_error', $buid, $str_errs ); + +} + +function getFilename_out( $buid, bool $bu_test, bool $isErr=false ){ + return getFilename_tmp( $buid, $bu_test?'t':'n', $isErr? 'err': 'op' ); +} + +/** + * Create an output file for the backup (forked) process to use. These are essentually + * temporary files which may be deleted after the back process completes. + * + * buid: backup record id + * + * fn_type: + * t: for test files + * n: for non-test files + * x: for file exclusion list + * + * fn_stream: + * op: output stream + * err: error stream + */ +function getFilename_tmp( $buid, $fn_type='t', $fn_stream='op' ){ + return "bu-$fn_type-$fn_stream-$buid.txt"; +} + + + +?> diff --git a/html/svc/db-writes.php b/html/svc/db-writes.php index 45af47d..d8e986f 100644 --- a/html/svc/db-writes.php +++ b/html/svc/db-writes.php @@ -89,13 +89,60 @@ */ function set_bu_running_db( $id, bool $setit ){ - global $g_database_path; - + global $g_database_path; $db = new SQLite3( $g_database_path ); $setit_int = $setit?1:0; return $db->exec("update BULIST set BuRunning=$setit_int where BUID=$id"); } +function set_last_run_date_db( $buid, $str_dt ){ + + global $g_database_path; + $db = new SQLite3( $g_database_path ); + + $p = $db->prepare("update BULIST set LastRunDt=:str_dt where BUID=:buid"); + $p->bindValue(':str_dt', $str_dt ); + $p->bindValue(':buid', $buid ); + $p->execute(); + + if( $db->lastErrorCode()!==0){ + return $db->lastErrorCode(); + } + return 0; +} + + +function set_sched_date_db( $buid, $dt_new ){ + + global $g_database_path; + $db = new SQLite3( $g_database_path ); + + $p = $db->prepare("update BULIST set BU_DATE=:str_dt where BUID=:buid"); + $p->bindValue(':str_dt', $dt_new ); + $p->bindValue(':buid', $buid ); + $p->execute(); + + if( $db->lastErrorCode()!==0){ + return $db->lastErrorCode(); + } + return 0; +} + +function delete_bu_item_db( $id ){ + + global $g_database_path; + $db = new SQLite3( $g_database_path ); + + $db_st = $db->prepare('delete from BULIST where BUID=:buid'); + $db_st->bindValue(':buid', $id ); + $db_st->execute(); + + if( $db->lastErrorCode()!==0){ + return $db->lastErrorCode(); + } + +} + ?> diff --git a/html/svc/main.php b/html/svc/main.php index 8f81018..d37c443 100644 --- a/html/svc/main.php +++ b/html/svc/main.php @@ -13,14 +13,21 @@ */ include __DIR__."/../lib/sysVcom.php"; +function debug_print( $msg ){ + print( $msg ); +} + //create the comms file touch('BU-commander'); //remove all queues at startup clear_IPC( '../svc/BU-commander' ); -rpc_setGlobals( 'procName', '../svc/BU-commander' ); +rpc_setGlobals( 'procName', 'BU-commander' ); rpc_setGlobals( 'max_chan', 8182 ); +rpc_setGlobals( 'main_pid', getmypid() ); + +print("rpc using ". __DIR__ . "/$g_procName with key = $g_k_caller\n" ); // problem is that the paths are not correct wrt this location, so we need to adjust them include __DIR__."/../inc/paths.php"; @@ -42,7 +49,7 @@ while(true){ $bDidWork = false; - + $bDidWork |= check_rpc_clients(); $bDidWork |= check_bu_sched(); @@ -65,6 +72,16 @@ } /** + * Make sure that the main table data is populated + */ +function ensure_cached_table(){ + global $g_bu_table; + if(count($g_bu_table)>0) return; + $colIdxs = null; //we don't ever reload these + getbu_list_all( $g_bu_table, $colIdxs, 'order by BU_DATE desc, BU_TIME desc' ); +} + +/** * Return true if the backup was kicked off */ function check_bu_sched(){ @@ -93,10 +110,10 @@ //...no it's not return; } - + kickoff_backup( $idfound ); - print( "check after forked: running = ". $g_bu_table[$idfound][ $g_bu_colIdxs['BuRunning'] ] . "\n"); + //print( "check after forked: running = ". $g_bu_table[$idfound][ $g_bu_colIdxs['BuRunning'] ] . "\n"); return true; } @@ -110,6 +127,8 @@ */ function kickoff_backup( $buid, $scheduled = true, $bu_test = false ){ + ensure_cached_table(); + global $g_bu_table; global $g_bu_colIdxs; @@ -146,7 +165,7 @@ print( "childpid = $pid\n" ); print( "kickoff_backup for $buid\n" ); - include_once __DIR__."/../inc/bu-common.php"; + include_once __DIR__."/backup.php"; $bu_result = ''; global $g_data_dir; $g_data_dir = __DIR__."/../data"; @@ -175,15 +194,10 @@ $caller_id = 0; - //don't reuse the local variable here, it gets corrupted. No idea why. - //keep separate in/out copies. $obj_in = rpc_listen( $caller_id, $msgArr, false ); $obj_out = null; - if($caller_id==0){ - //nothing to do - return false; - } + if($caller_id==0) return false; //nothing to do try{ $obj_out = handle_rpc_req( $obj_in ); @@ -192,26 +206,21 @@ print( "Exception: " . $e->getMessage() ."\n" ); $obj_out = array( 'e:Exception', $e ); } - + //send msg back rpc_reply( $caller_id, $obj_out ); - //done with the global message - unset( $msgArr[$caller_id] ); - - if( $obj_in[0] == 'm:fork-complete' ){ - if( count($obj_in)<5){ - //some bizzare things going on with the obj_in - print("m:fork-complete only has ". count($obj_in) . " params, should be 5, skipping client cleanup\n"); - print_r( $obj_in ); - return true; - } - print("now wait for child pid = {$obj_in[1]} \n"); - // this is required to clean up the child process + if( $obj_in[0] === 'm:fork-complete' ){ + //clean up child process + + print("clean up fork child pid = {$obj_in[1]} \n"); $status; pcntl_waitpid( $obj_in[1], $status ); - if( $obj_in[2]="backup-complete"){ + print("clean up done pid = {$obj_in[1]} \n"); + + //finish any requirements for specific child processes (db updates etc) + if( count($obj_in)>2 && $obj_in[2]="backup-complete"){ backupComplete( $obj_in[3], $obj_in[4] ); } } @@ -233,8 +242,6 @@ $nothanks = null; getbu_list_all( $g_bu_table, $nothanks, 'order by BU_DATE desc, BU_TIME desc' ); - include_once __DIR__."/../inc/bu-common.php"; - if($scheduled){ //if DB error, do nothing $err = $g_bu_table[$buid][ $g_bu_colIdxs['BuError'] ]; @@ -265,17 +272,13 @@ } $g_bu_table[$buid][ $g_bu_colIdxs['BU_DATE'] ] = $dt_new; - set_sched_date( $buid, $dt_new ); + set_sched_date_db( $buid, $dt_new ); } }//if manually kicked off - nothing to do //TODO: remove the outputfiles - $dt = gmdate("Y-m-d H:i:s"); - set_last_run_date( $buid, $dt ); - $g_bu_table[$buid][ $g_bu_colIdxs['LastRunDt'] ] = $dt; - } @@ -305,6 +308,7 @@ case 'f:getbu_list_state': global $g_bu_states; if($g_bu_states==null){ + //print("getbu_list_state (refresh\n"); $g_bu_states = getbu_list_state(); } return $g_bu_states; @@ -321,7 +325,7 @@ return array($err); case 'f:set_bu_error': - print("setting backup error\n"); + //print("re/setting backup error\n"); $err = set_bu_error_db( $obj[1], $obj[2] ); clear_cached_data(); return array($err); @@ -338,6 +342,24 @@ //we've changed this slightly e:Error, ok:, done is not working yet return kickoff_backup( $buid, false, $isTest ); + case 'f:set_last_run_date': + //[$err] = rpc( 'f:set_last_run_date', $buid, $str_dt ); + $err = set_last_run_date_db( $obj[1], $obj[2] ); + clear_cached_data(); + return array($err); + + /*case 'f:set_sched_date': + $err = set_sched_date_db( $obj[1], $obj[2] ); + clear_cached_data(); + return array($err);*/ + + case 'f:delete_bu_item': + //[$err] = rpc( 'f:delete_bu_item', $buid ); + $err = delete_bu_item_db( $obj[1] ); + clear_cached_data(); + return array($err); + + } array_unshift( $obj, "Unknown Command" ); diff --git a/notes.txt b/notes.txt index ad31e99..2ccb8f2 100644 --- a/notes.txt +++ b/notes.txt @@ -1,40 +1,59 @@ -3 Dec 2024 ----------- +Testing Rqd +----------- - Reworking the project to use middleware as a services. (main.php) + * Run now + - failure report etc + - file type (incremental) + - db type (overwrite, and save last copy) + + * Create new bu - file type + - test output - Clients will now use rpc to main.php to update the DB. - - I will work through all files to ensure that everything has been correctly updated. - - - In progress - ----------- - - bu-common.php - set_last_run_date, set_sched_date - - - Files left to update - -------------------- + * Edit existing bu + - test output - delete-file-bu.php - common.php - paths.php + * Create new bu - DB type + - test output (might not be possible for DB) + + +File hierachy +------------- + + Service Only + (DB access, No rpc) + + ./svc/main.php + ./svc/db-reads.php + ./svc/db-writes.php - post_handler.php - service_calls.php - do we need this? - login.php - pagemap.php - secure.php + Service Child Process Only + (Both rpc and DB access) + ./svc/backup.php - db-site-insert-all.php + Shared + (No rpc, No DB) + ./inc/common.php (some old code needs to be removed) + ./lib/time.php + + Web-server Only + (rcp, No DB) + + ./inc/bu_list_content.php + ./inc/secure.php + ./inc/pagemap.php + ./index.php + ./phpinfo.php + ./post_handler.php + ./inc/new-file-bu.php + ./inc/login.php + ./inc/edit-sh-bu.php + + Libraries + (included as required) + ./lib/sysVcom.php + + ./inc/paths.php - Files done - ---------- - - bu_list_content.php - edit-sh-bu.php - new-file-bu.php - service_calls.php - remove ? + +