<?php

// This is a PHP script that handles offloading of bandwidth from a web

// server. It's a sort of poor-man's Akamai. It doesn't need anything

// terribly complex (Apache, PHP, a writable directory).

//

// It works like this:

// - You have a webserver with dynamic content, and static content that

// may change arbitrarily (i.e. - various users making changes to their

// homepages, etc). This server is under a lot of load, mostly from

// the static content, which tends to be big. There may be multiple virtual

// hosts on this machine. We call this the "base" server.

// - You have at least one other webserver that you can use to offload some

// of the bandwidth. We call this the "offload" server.

// - You set up an Apache module (mod_offload) on the first server.

// mod_offload inserts itself into the request chain, and decides if a

// given file is safe static content (real file, not a script/cgi, no

// password, etc). In those cases, it sends a 302 redirect, pointing the

// client to the offload server.

// - The offload server gets a request from the redirected client. It then

// sends an HTTP HEAD request for the file in question to the base server

// while the client waits. It decides if it has the right file based on

// the HEAD. If it does, it serves the cached file.

// - If the file is out of date, or doesn't exist on the offload server, it

// sends a regular HTTP request for it to the base server and

// begins caching it. While caching it, it also feeds it to the client

// that has been waiting.

// - If another request comes in while the file is being cached, it will

// stream what is already there from disk, and then continue to feed as

// the rest shows up.

// !!! FIXME: issues to work out.

// - Could have a partial file cached if server crashes or power goes out.

// Add a "cacher's process id" to the metadata, and have those feeding

// from the cache decide if this process died...if so, wipe the entry and

// recache it.

// - Need to have a way to clean out old files. If x.zip is on the base,

// gets cached, and then is deleted, it'll stay on the offload server

// forever. Getting a 404 from the HEAD request will clean it out, but

// the offload server needs to know to do that.

//

// Installation:

// You need PHP with --enable-sysvsem support. You should configure PHP to not

// have a time limit on script execution (max_execution_time setting, or

// just don't run this script in safe mode and it'll handle it). PHP for

// Windows currently doesn't support sysvsem, so until someone writes me

// a mutex implementation, we assume you'll use a Unix box for this script.

//

// You need Apache to push every web request to this script, presumably in a

// virtual host, if not the entire server.

//

// Assuming this script was at /www/scripts/index.php, you would want to add

// this to Apache's config:

//

// AliasMatch ^.*$ "/www/scripts/index.php"

//

// If you don't have control over the virtual host's config file, you can't

// use AliasMatch, but if you can put an .htaccess file in the root of the

// virtual host, you can get away with this:

//

// ErrorDocument 404 /index.php

//

// This will make all missing files (everything) run the script, which will

// then cache and distribute the correct content, including overriding the

// 404 status code with the correct one. Be careful about files that DO exist

// in that vhost directory, though. They won't offload.

// You can offload multiple base servers with one box: set up one virtual host

// on the offload server for each base server. This lets each base server

// have its own cache and configuration.

//

// Then edit offload_server_config.php to fit your needs.

//

// Restart the server so the AliasMatch configuration tweak is picked up.

//

// This file is written by Ryan C. Gordon (icculus@icculus.org).

require_once 'PEAR.php';

$Guri = $_SERVER['REQUEST_URI'];

if (strcmp($Guri{0}, '/') != 0)

$Guri = '/' . $Guri;

$GFilePath = NULL;

$GMetaDataPath = NULL;

$GSemaphore = NULL;

$GSemaphoreOwned = 0;

$GLockDir = GOFFLOADDIR . '/lock-';

$GEtagFname = NULL;

function getDebugFilePointer()

{

global $GDebugFilePointer;

if ((!GDEBUG) || (!GDEBUGTOFILE))

return(NULL);

if (!isset($GDebugFilePointer))

if ($GDebugFilePointer === false)

$GDebugFilePointer = NULL;

} // if

return($GDebugFilePointer);

} // getDebugFilePointer

function debugEcho($str)

{

if (GDEBUG)

{

if (!is_array($str))

$str = $str . "\n";

if (!GDEBUGTOFILE)

print($str);

{

$fp = getDebugFilePointer();

if (isset($fp))

{

@fputs($fp, print_r($str, true));

@fflush($fp);

} // if

} // else

} // if

} // debugEcho

function etagToCacheFname($etag)

{

return(trim($etag, " \t\n\r\0\x0B\"'"));

} // etagToCacheFname

function getSemaphore()

{

debugEcho("grabbing semaphore...(owned $GSemaphoreOwned time(s).)");

if ($GSemaphoreOwned++ > 0)

return;

if (!isset($GSemaphore))

{

debugEcho('(have to create semaphore...)');

$GSemaphore = sem_get(0x8267bc62); // !!! FIXME: good value?

if ($GSemaphore === false)

failure('503 Service Unavailable', "Couldn't allocate semaphore.");

} // if

sem_acquire($GSemaphore);

else

{

if ($GEtagFname == NULL)

failure('503 Service Unavailable', 'Semaphore init failed');

$dir = $GLockDir . $GEtagFname;

$max = 100;

$count = 0;

while (($count < $max) && (@mkdir($dir) === false))

{

usleep(10000);

$count++;

} // while

if ($count == $max) // didn't get lock...force it. So nasty.

{

@rmdir($dir);

$GSemaphoreOwned--;

getSemaphore();

} // if

} // else

} // getSemaphore

function putSemaphore()

{

global $GSemaphore, $GSemaphoreOwned, $GLockDir, $GEtagFname;

if ($GSemaphoreOwned == 0)

return;

if (--$GSemaphoreOwned == 0)

{

if (GUSESEMAPHORE)

{

if (isset($GSemaphore))

sem_release($GSemaphore);

} // if

else

{

if ($GEtagFname != NULL)

@rmdir($GLockDir . $GEtagFname);

} // else

} // if

debugEcho("released semaphore...(now owned $GSemaphoreOwned time(s).)");

} // putSemaphore

function terminate()

{

debugEcho('offload script is terminating...');

while ($GSemaphoreOwned > 0)

putSemaphore();

if (isset($GDebugFilePointer))

exit();

} // terminate

function doHeader($str)

{

{

header($str, true);

if (headers_sent($filename, $linenum))

debugEcho("Headers already sent in $filename on line $linenum");

}

debugEcho("header('$str');");

} // doHeader

function sanestrpos($haystack, $needle)

{

$rc = strpos($haystack, $needle);

return(($rc === false) ? -1 : $rc);

} // sanestrpos

function loadMetadata($fname)

{

$retval = array();

$lines = @file($fname);

if ($lines === false)

return($retval);

$max = count($lines);

for ($i = 0; $i < $max; $i += 2)

{

$key = trim($lines[$i]);

$val = trim($lines[$i+1]);

debugEcho("Loaded metadata '$key' => '$val'");

$retval[$key] = $val;

} // for

debugEcho("Loaded $max metadata pair(s).");

return($retval);

} // loadMetadata

function cachedMetadataMostRecent($metadata, $head)

{

global $GFilePath;

if (!isset($metadata['Content-Length']))

return(false);

if (!isset($metadata['ETag']))

return(false);

if (!isset($metadata['Last-Modified']))

return(false);

if (strcmp($metadata['Content-Length'], $head['Content-Length']) != 0)

return(false);

if (strcmp($metadata['ETag'], $head['ETag']) != 0)

return(false);

if (strcmp($metadata['Last-Modified'], $head['Last-Modified']) != 0)

{

if (!isset($metadata['X-Offload-Is-Weak']))

return(false);

if (($metadata['X-Offload-Is-Weak']) == 0)

return(false);

} // if

// See if file size != Content-Length, and if it isn't,

// see if X-Offload-Caching-PID still exists. If process

// is missing, assume transfer died and recache.

$stat = @stat($GFilePath);

if ($stat === false)

return(false);

$fsize = $stat['size'];

if ($fsize != $metadata['Content-Length'])

{

// whoa, we were supposed to cache this!

if ($metadata['X-Offload-Caching-PID'] == getmypid())

return(false);

else if ($metadata['X-Offload-Caching-PID'] <= 0)

return(false);

// !!! FIXME: Unix specific!

if (!posix_kill($metadata['X-Offload-Caching-PID'], 0))

debugEcho('Caching process ID died!');

return(false);

} // if

} // if

return(true);

} // cachedMetadataMostRecent

function nukeRequestFromCache()

{

global $GMetaDataPath, $GFilePath;

debugEcho('Nuking request from cache...');

getSemaphore();

if (isset($GMetaDataPath))

@unlink($GMetaDataPath);

if (isset($GFilePath))

@unlink($GFilePath);

putSemaphore();

} // nukeRequestFromCache

function failure($httperr, $errmsg, $location = NULL)

{

global $GServerString;

if (strncasecmp($httperr, 'HTTP', 4) == 0)

{

$pos = sanestrpos($httperr, ' ');

if ($pos >= 0)

$httperr = substr($httperr, $pos+1);

} // if

$responseStr = "HTTP/1.0 $httperr";

debugEcho('failure() called:');

debugEcho(' ' . $httperr);

debugEcho(' ' . $errmsg);

doHeader($responseStr);

doHeader('Server: ' . $GServerString);

doHeader('Date: ' . HTTP::date());

if (isset($location))

doHeader('Location: ' . $location);

doHeader('Connection: close');

print("$errmsg\n");

terminate();

} // failure

function invalidContentRange($startRange, $endRange, $max)

{

if (($startRange < 0) || ($startRange >= $max))

return(true);

return(false);

} // invalidContentRange

function microtime_float()

{

list($usec, $sec) = explode(" ", microtime());

return ((float)$usec + (float)$sec);

} // microtime_float

function stopwatch($id = NULL)

{

static $storedid = NULL;

static $tod = NULL;

if (!GDEBUG)

return;

$now = microtime_float();

if (isset($id))

$storedid = $id;

if (!isset($tod))

$tod = $now;

else

{

debugEcho("Stopwatch [$storedid]: " . ($now - $tod) . ' seconds.');

$tod = NULL;

} // else

} // stopwatch

// error handler function

function myErrorHandler($errno, $errstr, $errfile, $errline)

{

switch ($errno)

{

case E_USER_ERROR:

debugEcho("PHP ERROR TRIGGERED: [$errno] $errstr");

debugEcho(" Fatal error in line $errline of file $errfile");

debugEcho(", PHP " . PHP_VERSION . " (" . PHP_OS . ")");

debugEcho("Aborting...");

exit(1);

break;

case E_USER_WARNING:

break;

case E_USER_NOTICE:

debugEcho("PHP NOTICE TRIGGERED:</b> [$errno] $errstr");

break;

default:

debugEcho("Unknown PHP error triggered!: [$errno] $errstr");

break;

} // switch

} // myErrorHandler

function debugInit()

{

global $Guri;

if (GDEBUG)

{

debugEcho('');

debugEcho('');

debugEcho('');

debugEcho('Offload Debug Run!');

debugEcho('');

debugEcho('User wants to get: ' . $Guri);

debugEcho('Request from address: ' . $_SERVER['REMOTE_ADDR'] . '.');

debugEcho('Client User-Agent: "' . $_SERVER['HTTP_USER_AGENT'] . '".');

debugEcho('Referrer string: "' . $_SERVER['HTTP_REFERER'] . '".');

debugEcho('Timeout for HTTP HEAD request is ' . GTIMEOUT . '.');

debugEcho('Data cache goes in "' . GOFFLOADDIR . '".');

debugEcho('');

debugEcho('');

} // if

// force PHP errors to not go through debug system and not to user.

error_reporting(E_USER_ERROR | E_USER_WARNING | E_USER_NOTICE);

set_error_handler('myErrorHandler');

} // debugInit

// The mainline...

debugInit();

// try to prevent script timeout.

set_time_limit(0);

failure('200 OK', "User-agent: *\nDisallow: /");

if (sanestrpos($Guri, '?') >= 0)

failure('403 Forbidden', "Offload server doesn't do dynamic content.");

$reqmethod = $_SERVER['REDIRECT_REQUEST_METHOD'];

if (!isset($reqmethod)

$reqmethod = $_SERVER['REQUEST_METHOD'];

if (!isset($reqmethod)

$reqmethod = 'GET';

$ishead = (strcasecmp($reqmethod, 'HEAD') == 0);

$isget = (strcasecmp($reqmethod, 'GET') == 0);

if ((!ishead) && (!isget))

failure('403 Forbidden', "Offload server doesn't do dynamic content.");

$origurl = 'http://' . GBASESERVER . $Guri;

stopwatch('HEAD transaction');

$head = HTTP::head($origurl, GTIMEOUT);

stopwatch();

if (PEAR::isError($head))

failure('503 Service Unavailable', 'Error: ' . $head->getMessage());

debugEcho($head);

if (($head['response_code'] == 401) || (isset($head['WWW-Authenticate'])))

failure('403 Forbidden', "Offload server doesn't do protected content.");

else if ($head['response_code'] != 200)

failure($head['response'], $head['response'], $head['Location']);

if ( (!isset($head['ETag'])) ||

(!isset($head['Content-Length'])) ||

(!isset($head['Last-Modified'])) )

{

failure('403 Forbidden', "Offload server doesn't do dynamic content.");

} // if

if (strlen($head['ETag']) > 2)

{

// a "weak" ETag?

if (strncasecmp($head['ETag'], "W/", 2) == 0)

{

debugEcho('Chopped ETag to be [' . $head['ETag'] . ']');

} // if

} // if

// !!! FIXME: Check Cache-Control, Pragma no-cache

$cacheio = NULL; // will be non-NULL if we're WRITING to the cache...

$frombaseserver = false;

$io = NULL; // read from this. May be file or HTTP connection.

// HTTP HEAD requests for PHP scripts otherwise run fully and throw away the

// results: http://www.figby.com/archives/2004/06/01/2004-06-01-php/

if ($ishead)

debugEcho('This is a HEAD request to the offload server.');

// Partial content:

// Does client want a range (download resume, "web accelerators", etc)?

$max = $head['Content-Length'];

$startRange = 0;

$endRange = $max-1;

if (isset($HTTP_SERVER_VARS['HTTP_IF_RANGE']))

{

// !!! FIXME: handle this.

$ifrange = $HTTP_SERVER_VARS['HTTP_IF_RANGE'];

debugEcho("Client set If-Range: [$ifrange]...unsupported!");

if (isset($HTTP_SERVER_VARS['HTTP_RANGE']))

unset($HTTP_SERVER_VARS['HTTP_RANGE']);

} // if

if (isset($HTTP_SERVER_VARS['HTTP_RANGE']))

{

$range = $HTTP_SERVER_VARS['HTTP_RANGE'];

debugEcho("There's a HTTP_RANGE specified: [$range].");

if (strncasecmp($range, 'bytes=', 6) != 0)

failure('400 Bad Request', 'Only ranges of "bytes" accepted.');

else if (strpos($range, ',') !== false)

failure('400 Bad Request', 'Multiple ranges not currently supported');

else

{

$range = substr($range, 6);

$pos = strpos($range, '-');

if ($pos !== false)

{

$startRange = trim(substr($range, 0, $pos));

$endRange = trim(substr($range, $pos + 1));

if (strcmp($startRange, '') == 0)

$startRange = 0;

if (strcmp($endRange, '') == 0)

$endRange = $max-1;

$reportRange = 1;

} // if

} // else

} // if

if ($endRange >= $max) // apparently, this is legal to request.

$endRange = $max - 1;

if (invalidContentRange($startRange, $endRange, $max))

failure('400 Bad Request', 'Bad content range requested.');

$GEtagFname = etagToCacheFname($head['ETag']);

$GFilePath = GOFFLOADDIR . '/filedata-' . $GEtagFname;

$GMetaDataPath = GOFFLOADDIR . '/metadata-' . $GEtagFname;

debugEcho('metadata cache is ' . $GMetaDataPath);

debugEcho('file cache is ' . $GFilePath);

if ($ishead)

$metadata = $head;

else

$metadata = loadMetadata($GMetaDataPath);

if (cachedMetadataMostRecent($metadata, $head))

{

$io = @fopen($GFilePath, 'rb');

if ($io === false)

failure('500 Internal Server Error', "Couldn't access cached data.");

debugEcho('File is cached.');

} // else if

else

{

// we need to pull a new copy from the base server...

$io = NULL;

$getheaders = HTTP::get($io, $origurl, GTIMEOUT); // !!! FIXME: may block, don't hold semaphore here!

if ($io === false)

failure('503 Service Unavailable', "Couldn't stream file to cache.");

stream_set_blocking($io, false);

stream_set_timeout($io, 60);

$cacheio = @fopen($GFilePath, 'wb');

if ($cacheio === false)

{

fclose($io);

failure('500 Internal Server Error', "Couldn't update cached data.");

} // if

$metaout = @fopen($GMetaDataPath, 'wb');

if ($metaout === false)

{

fclose($cacheio);

fclose($io);

nukeRequestFromCache();

failure('500 Internal Server Error', "Couldn't update metadata.");

} // if

// !!! FIXME: This is a race condition...may change between HEAD

// !!! FIXME: request and actual HTTP grab. We should really

// !!! FIXME: just use this for comparison once, and if we are

// !!! FIXME: recaching, throw this out and use the headers from the

// !!! FIXME: actual HTTP grab when really updating the metadata.

//

// !!! FIXME: Also, write to temp file and rename in case of write failure!

if (!isset($head['Content-Type'])) // make sure this is sane.

$head['Content-Type'] = 'application/octet-stream';

$head['X-Offload-Caching-PID'] = getmypid();

foreach ($head as $key => $val)

fputs($metaout, $key . "\n" . $val . "\n");

fclose($metaout);

$metadata = $head;

debugEcho('Cache needs refresh...pulling from base server...');

} // else

putSemaphore();

} // else

doHeader('Date: ' . HTTP::date());

doHeader('Server: ' . $GServerString);

doHeader('Connection: close');

doHeader('ETag: ' . $metadata['ETag']);

doHeader('Last-Modified: ' . $metadata['Last-Modified']);

if ($reportRange)

doHeader("Content-Range: bytes $startRange-$endRange/$max");

if ($ishead)

{

debugEcho('This was a HEAD request to offload server, so it is done.');

$br = 0;

$readsize = $startRange - $br;

if (($readsize <= 0) || ($readsize > 8192))

$readsize = 8192;

if ($readsize > ($endRange - $br))

$readsize = ($endRange - $br);

if ($readsize == 0)

break; // Shouldn't hit, but just in case...

if (feof($io))

{

debugEcho('feof() triggered.');

break;

} // if

if ($frombaseserver)

{

$info = stream_get_meta_data($io);

if ($info['eof'])

{

debugEcho('socket meta data has eof flag.');

break;

} // if

else if ($info['timed_out'])

{

debugEcho('socket meta data has timed_out flag.');

break;

} // if

} // if

else

{

$stat = @fstat($io);

if ($stat === false)

break;

$cursize = $stat['size'];

if ($cursize < $max)

{

{

sleep(1);

continue;

} // if

} // if

} // if

$len = strlen($data);

if ($len > 0)

if (isset($cacheio))

{

fwrite($cacheio, $data); // !!! FIXME: check for errors!

fflush($cacheio);

} // if

if (!connection_aborted())

{

if ((!GDEBUG) || (GDEBUGTOFILE))

{

if (($br >= $startRange) && ($br < $endRange))

{

$verb = GDEBUGTOFILE ? 'Wrote ' : 'Would have written ';

debugEcho($verb . $len . ' bytes.');

// If this connection is cacheing from base server, we have to keep going.

if (($br == $endRange) && (isset($cacheio)) && ($br != $max))

{

debugEcho('Sent complete request, but am pulling from base server!');

$endRange = $max;

} // if

} // while

debugEcho('Transfer is complete.');

if (isset($cacheio))

@fclose($cacheio);

if ($br != $endRange)

if ($frombaseserver)

nukeRequestFromCache();

} // if

terminate();

// end of offload script ...

// This is HTTP from PEAR. Copied here for my convenience.

// I trimmed some stuff out and hacked on some other code.

// --ryan.

class HTTP

{

{

if (!isset($time)) {

$time = time();

} elseif (!is_numeric($time) && (-1 === $time = strtotime($time))) {

return(false);

}

// RFC822 or RFC850

$format = ini_get('y2k_compliance') ? 'D, d M Y' : 'l, d-M-y';

return gmdate($format .' H:i:s \G\M\T', $time);

}

function negotiateLanguage($supported, $default = 'en-US')

{

$supp = array();

foreach ($supported as $lang => $isSupported) {

if ($isSupported) {

$supp[strToLower($lang)] = $lang;

}

}

if (!count($supp)) {

return $default;

}

$matches = array();

if (isset($_SERVER['HTTP_ACCEPT_LANGUAGE'])) {

foreach (explode(',', $_SERVER['HTTP_ACCEPT_LANGUAGE']) as $lang) {

$lang = array_map('trim', explode(';', $lang));

if (isset($lang[1])) {

$l = strtolower($lang[0]);

$q = (float) str_replace('q=', '', $lang[1]);

} else {

$l = strtolower($lang[0]);

$q = null;

}

if (isset($supp[$l])) {

$matches[$l] = isset($q) ? $q : 1000 - count($matches);

}

}

}

if (count($matches)) {

asort($matches, SORT_NUMERIC);

return $supp[array_pop(array_keys($matches))];

}

if (isset($_SERVER['REMOTE_HOST'])) {

$lang = strtolower(array_pop(explode('.', $_SERVER['REMOTE_HOST'])));

if (isset($supp[$lang])) {

return $supp[$lang];

}

}

return $default;

}

{

$p = parse_url($url);

if (!isset($p['scheme'])) {

$p = parse_url(HTTP::absoluteURI($url));

} elseif ($p['scheme'] != 'http') {

return HTTP::raiseError('Unsupported protocol: '. $p['scheme']);

}

$port = isset($p['port']) ? $p['port'] : 80;

//debugEcho(array($p['host'], $port, $eno, $estr, $timeout));