/
mod_offload.c
509 lines (451 loc) · 19.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
/*
* mod_offload
*
* This is a module that lets you redirect static content to "offload"
* servers, such that they can distribute the bandwidth load normally
* given to the "base" server. For sites that are completely static and/or
* completely replicated via other methods such as rsync, installing this
* module may be sufficient to spread the burden between multiple servers.
* This can be paired with the included offload.php script that aids in
* keeping servers in sync without the need to copy the entire base server's
* contents or install other tools and daemons on the offload servers
* (just this module on the base server, and a PHP script on the offload
* servers). icculus.org uses this to spread the bandwidth around.
*
* To Build/Install as a shared module:
* /where/i/installed/apache/bin/apxs -c -i -a mod_offload.c
*
* Make sure this is in your Apache config:
* LoadModule offload_module libexec/mod_offload.so
* AddModule mod_offload.c
*
* Now pick a directory/location/virtualhost/htaccess and add some directives:
*
* OffloadEngine On
* OffloadHost offload1.myhost.com
*
* That's all you need for basic operation. This enables offloading, and
* sets up one hostname ("offload1.myhost.com") as an offload server.
*
* All directives are listed below. They may appear anywhere in the
* server-wide configuration files (e.g., httpd.conf), and in .htaccess
* files when the scope is covered by an AllowOverride Options keyword.
*
* OffloadEngine <On|Off>
* ...turn on or off the offload functionality. Defaults to Off.
* If not enabled, this module will just pass requests through to the
* next Apache handler untouched.
*
* OffloadHost <hostname>
* ...An offload host. You need at least one specified for offloading
* to function. You can specify this directive multiple times to add
* more offload servers. These can be IP addresses or FQDN's...this
* module will try to do a DNS lookup at startup if necessary, and will
* fail if it can't.
*
* OffloadDebug <On|Off>
* ...when on, will write details about every transaction to the error
* log. You want this turned off as soon as you are satisfied the
* module is functioning well.
*
* OffloadMinSize <number>
* ...files smaller than <number> are never offloaded, since in many
* cases it's just as easy to serve them from the base server without
* any load spike and less margin of error. This defaults to 5 kilobytes
* if not set, but can be set higher or lower. Zero will remove any
* minimum size check.
*
* OffloadExcludeMimeType <pattern>
* ...files with mimetypes matching <pattern> are never offloaded.
60
* This can be a wildcard pattern, so both "text/html" and "text/h*" are
61
62
* valid here.
*
63
64
65
66
* OffloadExcludeUserAgent <pattern>
* ...clients with User-Agent fields matching <pattern> are never offloaded.
* This can be a wildcard pattern.
*
67
68
69
70
71
72
73
74
75
76
77
78
79
* For URLs where mod_offload is in effect, it'll go through a checklist.
* Anything in the checklist that fails means that offloading shouldn't
* occur and Apache should handle this request as it would without
* mod_offload installed at all.
*
* - Is OffloadEngine On?
* - Is there at least one OffloadHost?
* - Is this a GET method (as opposed to HEAD, POST, etc)?
* - Is this a request without args (potentially static content)?
* - Is this the last Apache content handler (potentially static content)?
* - Is the desired file really there?
* - Is the desired file more than OffloadMinSize?
* - Is the request from someone other than an offload server?
80
* - Is the request not explicitly trying to bypass offloading?
81
* - Is the desired file's mimetype not listed in OffloadExcludeMimeType?
82
* - Is the client's User-Agent not listed in OffloadExcludeUserAgent?
83
84
85
86
87
88
89
90
91
92
93
*
* If the module makes it all the way through the checklist, it picks a
* random offload server (the server is chosen by the current
* time of day, with the chosen server changing once per second,
* rotating through the list of OffloadHost directives) and
* makes a 307 Redirect response to the client, pointing them to the
* offload server where they will receive the file.
*
* This file written by Ryan C. Gordon (icculus@icculus.org).
*/
94
#include "ap_config.h"
95
#include "httpd.h"
96
#include "http_request.h"
97
98
99
#include "http_config.h"
#include "http_core.h"
#include "http_log.h"
100
#include "http_main.h"
101
102
#include "http_protocol.h"
103
#define MOD_OFFLOAD_VER "1.0.2"
104
#define DEFAULT_MIN_OFFLOAD_SIZE (5 * 1024)
105
106
107
108
#define VERSION_COMPONENT "mod_offload/"MOD_OFFLOAD_VER
/* some Apache 1.3/2.0 compatibility glue... */
#ifndef STANDARD20_MODULE_STUFF
109
module MODULE_VAR_EXPORT offload_module;
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
# define TARGET_APACHE_1_3 1
# include "fnmatch.h"
# define wild_match(p,s) (ap_fnmatch(p, s, FNM_CASE_BLIND) == 0)
# define REQ_AUTH_TYPE(r) (r)->connection->ap_auth_type
# define FINFO_MODE(r) (r)->finfo.st_mode
# define FINFO_SIZE(r) (r)->finfo.st_size
# define apr_vsnprintf(a,b,c,d) ap_vsnprintf(a,b,c,d)
# define apr_table_get(a,b) ap_table_get(a,b)
# define apr_table_setn(a,b,c) ap_table_setn(a,b,c)
# define apr_pstrcat(a,b,c,d,e) ap_pstrcat(a,b,c,d,e)
# define apr_pstrdup(a,b) ap_pstrdup(a,b)
# define apr_palloc(a,b) ap_palloc(a,b)
# define apr_array_make(a,b,c) ap_make_array(a,b,c)
# define apr_array_push(a) ap_push_array(a)
# define AP_INIT_FLAG(a,b,c,d,e) { a,b,c,d,FLAG,e }
# define AP_INIT_TAKE1(a,b,c,d,e) { a,b,c,d,TAKE1,e }
typedef struct in_addr apr_sockaddr_t;
typedef array_header apr_array_header_t;
typedef pool apr_pool_t;
typedef table apr_table_t;
#else
131
module AP_MODULE_DECLARE_DATA offload_module;
132
133
134
135
136
137
138
139
140
141
# include "apr_strings.h"
# include "apr_lib.h"
# include "apr_fnmatch.h"
# define wild_match(p,s) (apr_fnmatch(p,s,APR_FNM_CASE_BLIND) == APR_SUCCESS)
# define REQ_AUTH_TYPE(r) (r)->ap_auth_type
# define FINFO_MODE(r) (r)->finfo.protection
# define FINFO_SIZE(r) (r)->finfo.size
#endif
142
143
144
145
146
typedef struct
{
int offload_engine_on;
int offload_debug;
int offload_min_size;
147
148
149
150
apr_array_header_t *offload_hosts;
apr_array_header_t *offload_ips;
apr_array_header_t *offload_exclude_mime;
apr_array_header_t *offload_exclude_agents;
151
152
153
} offload_dir_config;
154
static void debugLog(const request_rec *r, const offload_dir_config *cfg,
155
156
157
158
159
160
161
const char *fmt, ...)
{
if (cfg->offload_debug)
{
char buf[512];
va_list ap;
va_start(ap, fmt);
162
apr_vsnprintf(buf, sizeof (buf), fmt, ap);
163
va_end(ap);
164
165
166
167
168
169
ap_log_rerror(APLOG_MARK,
APLOG_NOERRNO|APLOG_ERR,
#if !TARGET_APACHE_1_3
APR_SUCCESS,
#endif
r, "mod_offload: %s", buf);
170
171
172
173
174
175
176
} /* if */
} /* debugLog */
static int offload_handler(request_rec *r)
{
int i = 0;
177
apr_sockaddr_t *list = NULL;
178
179
180
181
182
offload_dir_config *cfg = NULL;
char *uri = NULL;
int nelts = 0;
int idx = 0;
char *offload_host = NULL;
183
const char *user_agent = NULL;
184
const char *bypass = NULL;
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
cfg = (offload_dir_config *) ap_get_module_config(r->per_dir_config,
&offload_module);
/*
* All these DECLINED returns means "pass to the next handler", which
* may just be an immediate failure, 404, etc, but then it behaves how
* you'd expect your Apache to under normal circumstances.
*/
/* is OffloadEngine disabled? DECLINED */
if (!cfg->offload_engine_on) {
debugLog(r, cfg, "OffloadEngine is Off");
return DECLINED;
} /* if */
/* are there no offload servers defined? DECLINED */
nelts = cfg->offload_hosts->nelts;
if (nelts == 0) {
debugLog(r, cfg, "No offload hosts defined");
return DECLINED;
} /* if */
/* is it a HEAD request? DECLINED */
if (r->header_only) {
debugLog(r, cfg, "HEAD request for URI '%s'", r->unparsed_uri);
return DECLINED;
} /* if */
/* is it not a GET? DECLINED */
if (r->method_number != M_GET) {
debugLog(r, cfg, "Not a GET method for URI '%s'", r->unparsed_uri);
return DECLINED;
} /* if */
/* are there any args? DECLINED */
if (r->args != NULL) {
debugLog(r, cfg, "URI '%s' has args...dynamic?", r->unparsed_uri);
return DECLINED;
} /* if */
/* is there a password? DECLINED */
227
if (REQ_AUTH_TYPE(r) != NULL) {
228
229
230
231
debugLog(r, cfg, "URI '%s' requires auth", r->unparsed_uri);
return DECLINED;
} /* if */
232
#if TARGET_APACHE_1_3 /* we just insert as the last hook in 2.0 API. */
233
234
/* is there any dynamic content handler? DECLINED */
if (r->handler != NULL) {
235
236
debugLog(r, cfg, "URI '%s' has handler '%s'.",
r->unparsed_uri, r->handler);
237
238
return DECLINED;
} /* if */
239
#endif
240
241
/* is file missing? DECLINED */
242
if ((FINFO_MODE(r) == 0) || (r->path_info && *r->path_info)) {
243
244
245
246
247
debugLog(r, cfg, "File '%s' missing", r->unparsed_uri);
return DECLINED;
} /* if */
/* is file less than so-and-so? DECLINED */
248
if (FINFO_SIZE(r) < cfg->offload_min_size) {
249
debugLog(r, cfg, "File '%s' too small (%d is less than %d)",
250
251
r->unparsed_uri, (int) FINFO_SIZE(r),
(int) cfg->offload_min_size);
252
253
254
return DECLINED;
} /* if */
255
/* is this request from one of the listed offload servers? DECLINED */
256
list = (apr_sockaddr_t *) cfg->offload_ips->elts;
257
for (i = 0; i < cfg->offload_ips->nelts; i++) {
258
259
260
261
262
263
264
#if TARGET_APACHE_1_3
int match=(r->connection->remote_addr.sin_addr.s_addr==list[i].s_addr);
#else
int match = apr_sockaddr_equal(r->connection->remote_addr, &list[i]);
#endif
if (match)
{
265
266
267
268
269
270
271
offload_host = ((char **) cfg->offload_hosts->elts)[i];
debugLog(r, cfg, "Offload server (%s) doing cache refresh on '%s'",
offload_host, r->unparsed_uri);
return DECLINED;
} /* if */
} /* for */
272
273
274
275
276
277
278
279
/* Is this an explicit request to bypass offloading? DECLINED */
bypass = (const char *) apr_table_get(r->headers_in, "X-Mod-Offload-Bypass");
if (bypass) {
debugLog(r, cfg, "Client explicitly bypassing offloading for '%s'",
r->unparsed_uri);
return DECLINED;
} /* if */
280
281
282
283
284
285
/* is the file in the list of mimetypes to never offload? DECLINED */
if ((r->content_type) && (cfg->offload_exclude_mime->nelts))
{
for (i = 0; i < cfg->offload_exclude_mime->nelts; i++)
{
char *mimetype = ((char **) cfg->offload_exclude_mime->elts)[i];
286
287
if (wild_match(mimetype, r->content_type))
{
288
debugLog(r, cfg,
289
"URI '%s' (%s) is excluded from offloading"
290
291
292
293
294
295
296
" by mimetype pattern '%s'", r->unparsed_uri,
r->content_type, mimetype);
return DECLINED;
} /* if */
} /* for */
} /* if */
297
/* is this User-Agent excluded from offloading (like Google)? DECLINED */
298
user_agent = (const char *) apr_table_get(r->headers_in, "User-Agent");
299
300
301
302
303
if ((user_agent) && (cfg->offload_exclude_agents->nelts))
{
for (i = 0; i < cfg->offload_exclude_agents->nelts; i++)
{
char *agent = ((char **) cfg->offload_exclude_agents->elts)[i];
304
305
if (wild_match(agent, user_agent))
{
306
307
308
309
310
311
312
313
314
debugLog(r, cfg,
"URI request '%s' from agent '%s' is excluded from"
" offloading by User-Agent pattern '%s'",
r->unparsed_uri, user_agent, agent);
return DECLINED;
} /* if */
} /* for */
} /* if */
315
316
317
318
319
320
321
/* We can offload this. Pick a random offload servers from defined list. */
debugLog(r, cfg, "Offloading URI '%s'", r->unparsed_uri);
idx = (int)(time(NULL) % nelts);
offload_host = ((char **) cfg->offload_hosts->elts)[idx];
debugLog(r, cfg, "Chose server #%d (%s)", idx, offload_host);
/* Offload it: set a "Location:" header and 302 redirect. */
322
uri = apr_pstrcat(r->pool, "http://", offload_host, r->unparsed_uri, NULL);
323
324
debugLog(r, cfg, "Redirect from '%s' to '%s'", r->unparsed_uri, uri);
325
apr_table_setn(r->headers_out, "Location", uri);
326
327
328
329
return HTTP_TEMPORARY_REDIRECT;
} /* offload_handler */
330
static void *create_offload_dir_config(apr_pool_t *p, char *dummy)
331
332
{
offload_dir_config *retval =
333
(offload_dir_config *) apr_palloc(p, sizeof (offload_dir_config));
334
335
336
retval->offload_engine_on = 0;
retval->offload_debug = 0;
337
338
339
340
retval->offload_hosts = apr_array_make(p, 0, sizeof (char *));
retval->offload_exclude_mime = apr_array_make(p, 0, sizeof (char *));
retval->offload_exclude_agents = apr_array_make(p, 0, sizeof (char *));
retval->offload_ips = apr_array_make(p, 0, sizeof (apr_sockaddr_t));
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
retval->offload_min_size = DEFAULT_MIN_OFFLOAD_SIZE;
return retval;
} /* create_offload_dir_config */
static const char *offload_engine(cmd_parms *parms, void *mconfig, int flag)
{
offload_dir_config *cfg = (offload_dir_config *) mconfig;
cfg->offload_engine_on = flag;
return NULL; /* no error. */
} /* offload_engine */
static const char *offload_debug(cmd_parms *parms, void *mconfig, int flag)
{
offload_dir_config *cfg = (offload_dir_config *) mconfig;
cfg->offload_debug = flag;
return NULL; /* no error. */
} /* offload_debug */
static const char *offload_host(cmd_parms *parms, void *mconfig,
364
const char *_arg)
365
366
{
offload_dir_config *cfg = (offload_dir_config *) mconfig;
367
368
char **hostelem = (char **) apr_array_push(cfg->offload_hosts);
apr_sockaddr_t *addr = (apr_sockaddr_t *) apr_array_push(cfg->offload_ips);
369
370
371
372
373
374
375
char *ptr = NULL;
char arg[512];
apr_cpystrn(arg, _arg, sizeof (arg));
ptr = strchr(arg, ':');
if (ptr != NULL)
*ptr = '\0'; /* chop off port number if it's there. */
376
377
#if TARGET_APACHE_1_3
378
379
380
struct hostent *hp = ap_pgethostbyname(parms->pool, arg);
if (hp == NULL)
return "DNS lookup failure!";
381
382
383
384
385
386
387
388
389
memcpy(addr, (apr_sockaddr_t *) (hp->h_addr), sizeof (apr_sockaddr_t));
#else
apr_sockaddr_t *resolved = NULL;
apr_status_t rc;
rc = apr_sockaddr_info_get(&resolved, arg, APR_UNSPEC, 0, 0, parms->pool);
if (rc != APR_SUCCESS)
return "DNS lookup failure!";
memcpy(addr, resolved, sizeof (apr_sockaddr_t));
#endif
390
391
*hostelem = apr_pstrdup(parms->pool, _arg);
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
return NULL; /* no error. */
} /* offload_host */
static const char *offload_minsize(cmd_parms *parms, void *mconfig,
const char *arg)
{
offload_dir_config *cfg = (offload_dir_config *) mconfig;
cfg->offload_min_size = atoi(arg);
return NULL; /* no error. */
} /* offload_minsize */
static const char *offload_excludemime(cmd_parms *parms, void *mconfig,
const char *arg)
{
offload_dir_config *cfg = (offload_dir_config *) mconfig;
409
410
char **mimepattern = (char **) apr_array_push(cfg->offload_exclude_mime);
*mimepattern = apr_pstrdup(parms->pool, arg);
411
return NULL; /* no error. */
412
413
414
415
416
417
418
} /* offload_excludemime */
static const char *offload_excludeagent(cmd_parms *parms, void *mconfig,
const char *arg)
{
offload_dir_config *cfg = (offload_dir_config *) mconfig;
419
420
char **agentpattern = (char **) apr_array_push(cfg->offload_exclude_agents);
*agentpattern = apr_pstrdup(parms->pool, arg);
421
422
return NULL; /* no error. */
} /* offload_excludeagent */
423
424
425
static const command_rec offload_cmds[] =
426
{
427
428
429
430
431
432
433
434
435
436
437
438
439
AP_INIT_FLAG("OffloadEngine", offload_engine, NULL, OR_OPTIONS,
"Set to On or Off to enable or disable offloading"),
AP_INIT_FLAG("OffloadDebug", offload_debug, NULL, OR_OPTIONS,
"Set to On or Off to enable or disable debug spam to error log"),
AP_INIT_TAKE1("OffloadHost", offload_host, NULL, OR_OPTIONS,
"Hostname or IP address of offload server"),
AP_INIT_TAKE1("OffloadMinSize", offload_minsize, NULL, OR_OPTIONS,
"Minimum size, in bytes, that a file must be to be offloaded"),
AP_INIT_TAKE1("OffloadExcludeMimeType",offload_excludemime,0,OR_OPTIONS,
"Mimetype to always exclude from offloading (wildcards allowed)"),
AP_INIT_TAKE1("OffloadExcludeUserAgent",offload_excludeagent,0,OR_OPTIONS,
"User-Agent to always exclude from offloading (wildcards allowed)"),
{ NULL }
440
441
442
};
443
444
445
446
447
448
449
/* Tell Apache what phases of the transaction we handle */
#if TARGET_APACHE_1_3
static void init_offload(server_rec *s, apr_pool_t *p)
{
ap_add_version_component(VERSION_COMPONENT);
} /* init_offload */
450
451
452
453
454
455
456
457
/* Make the name of the content handler known to Apache */
static handler_rec offload_handlers[] =
{
{ "*/*", offload_handler },
{"offload-handler", offload_handler},
{ NULL , NULL }
};
458
module MODULE_VAR_EXPORT offload_module =
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
{
STANDARD_MODULE_STUFF,
init_offload, /* module initializer */
create_offload_dir_config, /* per-directory config creator */
NULL, /* dir config merger */
NULL, /* server config creator */
NULL, /* server config merger */
offload_cmds, /* command table */
offload_handlers, /* [9] content handlers */
NULL, /* [2] URI-to-filename translation */
NULL, /* [5] check/validate user_id */
NULL, /* [6] check user_id is valid *here* */
NULL, /* [4] check access by host address */
NULL, /* [7] MIME type checker/setter */
NULL, /* [8] fixups */
NULL, /* [10] logger */
NULL, /* [3] header parser */
NULL, /* process initialization */
NULL, /* process exit/cleanup */
NULL /* [1] post read_request handling */
};
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
#else /* Apache 2.0 module API ... */
int offload_init(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp,
server_rec *base_server)
{
ap_add_version_component(p, VERSION_COMPONENT);
return OK;
} /* init_offload */
static void offload_register_hooks(apr_pool_t *p)
{
ap_hook_post_config(offload_init, NULL, NULL, APR_HOOK_MIDDLE);
ap_hook_handler(offload_handler, NULL, NULL, APR_HOOK_LAST);
} /* offload_register_hooks */
module AP_MODULE_DECLARE_DATA offload_module =
{
STANDARD20_MODULE_STUFF,
create_offload_dir_config, /* create per-directory config structures */
NULL, /* merge per-directory config structures */
NULL, /* create per-server config structures */
NULL, /* merge per-server config structures */
offload_cmds, /* command handlers */
offload_register_hooks /* register hooks */
};
#endif
508
/* end of mod_offload.c ... */