:pserver:anonymous@cvs.middle-man.sourceforge.net:/cvsroot/middle-man middleman
[middleman.git] / src / protocol.c
1 /*
2     MiddleMan filtering proxy server
3     Copyright (C) 2002  Jason McLaughlin
4
5     This program is free software; you can redistribute it and/or modify
6     it under the terms of the GNU General Public License as published by
7     the Free Software Foundation; either version 2 of the License, or
8     (at your option) any later version.
9
10     This program is distributed in the hope that it will be useful,
11     but WITHOUT ANY WARRANTY; without even the implied warranty of
12     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13     GNU General Public License for more details.
14
15     You should have received a copy of the GNU General Public License
16     along with this program; if not, write to the Free Software
17     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18 */
19
20 #include <stdio.h>
21 #include <string.h>
22 #include <stdlib.h>
23 #include <unistd.h>
24 #include <sys/types.h>
25 #include <sys/time.h>
26 #include "../libntlm/ntlm.h"
27 #include "proto.h"
28
29 extern TEMPLATES *templates;
30 extern REWRITE_LIST *rewrite_list;
31 extern EXTERNAL *external;
32 extern MIME_LIST *mime_list;
33 extern REDIRECT_LIST *redirect_list;
34 extern KEYWORD_LIST *keyword_list;
35 extern THREADLIST threads[];
36
37 int protocol_start(CONNECTION * connection)
38 {
39         int proxyfd, x;
40
41         switch (connection->proxy_type) {
42         case PROXY_NORMAL:
43                 proxyfd = net_connect(connection->proxy_host, connection->proxy_port);
44                 if (proxyfd < 0)
45                         putlog(MMLOG_ERROR, "connection to proxy server failed");
46                 else
47                         connection->server = sock_new(proxyfd);
48
49                 break;
50         case PROXY_SOCKS4:
51                 proxyfd = net_connect(connection->proxy_host, connection->proxy_port);
52
53                 if (proxyfd >= 0) {
54                         connection->server = sock_new(proxyfd);
55
56                         x = net_socks4(connection, connection->header->host, connection->header->port);
57                         if (x < 0) {
58                                 putlog(MMLOG_ERROR, "connection to socks4 server failed");
59
60                                 template_send(templates, error_to_template(x), connection, 404);
61
62                                 return -1;
63                         }
64                 }
65
66                 break;
67         default:
68                 proxyfd = net_connect(connection->header->host, connection->header->port);
69                 if (proxyfd >= 0)
70                         connection->server = sock_new(proxyfd);
71         }
72
73         if (proxyfd < 0) {
74                 template_send(templates, error_to_template(proxyfd), connection, 404);
75
76                 connection->server = NULL;
77
78                 return -1;
79         }
80
81         return proxyfd;
82 }
83
84 int protocol_reconnect(CONNECTION * connection)
85 {
86         int ret;
87
88         sock_close(connection->server);
89         sock_flush(connection->client);
90
91         ret = protocol_start(connection);
92
93         return ret;
94 }
95
96 /*
97 This function handles HTTP requests
98 */
99 int protocol_http(CONNECTION * connection)
100 {
101         int auth = FALSE, ret;
102         char *headbuf, buf[8096], *ptr;
103         FILEBUF *filebuf, *ext_filebuf, *postdata = NULL;
104         struct EXTERNAL_LIST_LIST *external_list = NULL;
105         struct MIME_LIST_LIST *mime_list_list;
106         struct url_command_t **url_command;
107
108         pthread_mutex_lock(&threads[connection->thread].lock);
109         threads[connection->thread].flags |= THREAD_HTTP;
110         pthread_mutex_unlock(&threads[connection->thread].lock);
111
112         if (connection->header->content_length != -1) {
113                 /* the browser wants to send something to the remote site after the header (POST, PUT, etc.) */
114                 putlog(MMLOG_DEBUG, "post length: %d", connection->header->content_length);
115
116                 /* unfortunately, this _needs_ to be buffered.. otherwise we won't have any POST data
117                    after authentication */
118                 postdata = http_transfer_filebuf(connection, CLIENT);
119                 rewrite_do(rewrite_list, connection, postdata, REWRITE_POST, TRUE);
120
121                 connection->header->chunked = FALSE;
122                 connection->header->content_length = postdata->size;
123         }
124
125         header_send(connection->header, connection, SERVER, (connection->proxy_type == PROXY_NORMAL) ? HEADER_FORWARD : HEADER_DIRECT);
126
127         /* I really hate using goto's */
128       auth_finished:
129         if (postdata != NULL)
130                 net_filebuf_send(postdata, connection, SERVER);
131
132       reget_header:
133         headbuf = header_get(connection, SERVER, TIMEOUT);
134         if (headbuf == NULL) {
135                 if (!connection->keepalive_server && auth == FALSE) {
136                         template_send(templates, "noconnect", connection, 503);
137
138                         if (postdata != NULL)
139                                 filebuf_free(postdata);
140
141                         return -1;
142                 } else {
143                         /* keep-alive connection closed, or authorization is being done on
144                            a non-keepalive proxy */
145                         ret = protocol_reconnect(connection);
146                         if (ret == -1) {
147                                 if (postdata != NULL)
148                                         filebuf_free(postdata);
149
150                                 return -1;
151                         }
152
153                         header_send(connection->header, connection, SERVER, (connection->proxy_type == PROXY_NORMAL) ? HEADER_FORWARD : HEADER_DIRECT);
154                         if (postdata != NULL)
155                                 net_filebuf_send(postdata, connection, SERVER);
156
157                         headbuf = header_get(connection, SERVER, TIMEOUT);
158                         if (headbuf == NULL) {
159                                 if (postdata != NULL)
160                                         filebuf_free(postdata);
161
162                                 return -1;
163                         }
164                 }
165         }
166
167         /* pass the server's header through the rewrite rules */
168         filebuf = filebuf_new();
169         filebuf->data = headbuf;
170         filebuf->size = strlen(headbuf) + 1;
171
172         rewrite_do(rewrite_list, connection, filebuf, REWRITE_SERVER, TRUE);
173
174         headbuf = filebuf->data;
175         filebuf->data = NULL;
176
177         filebuf_free(filebuf);
178
179         connection->rheader = http_header_parse_response(headbuf);
180         if (connection->rheader == NULL) {
181                 if (postdata != NULL)
182                         filebuf_free(postdata);
183
184                 template_send(templates, "badresponse", connection, 400);
185                 xfree(headbuf);
186                 return -1;
187         }
188         xfree(headbuf);
189
190         if (connection->rheader->code == 100) {
191                 /* server sent notice to continue POST, go back and retrieve final header 
192                    (according to the RFC, the server should not send this unless the client sends an Expect:
193                    header; however... IIS will sometimes send this anyways)
194                 */
195                 http_header_free(connection->rheader);
196
197                 goto reget_header;
198         }
199
200         connection->rheader->host = xstrdup(connection->header->host);
201         connection->rheader->file = xstrdup(connection->header->file);
202
203         if (connection->keepalive_client) {
204                 if (connection->rheader->content_length == -1 && !connection->rheader->chunked) {
205                         /* connection can't be kept alive if we can't determine when the remote site's data ends */
206                         connection->keepalive_client = FALSE;
207                         connection->keepalive_server = FALSE;
208                 } else if (connection->rheader->keepalive == TRUE || connection->rheader->proxy_keepalive == TRUE)
209                         connection->keepalive_server = TRUE;
210                 else if (connection->rheader->version == HTTP_HTTP11)
211                         /* keep-alive is assumed for HTTP/1.1 unless any of the above conditions are met */
212                         connection->keepalive_server = TRUE;
213                 else
214                         connection->keepalive_server = FALSE;
215
216                 putlog(MMLOG_DEBUG, "keepalive_server = %d", connection->keepalive_server);
217         } else
218                 connection->keepalive_server = FALSE;
219
220         if (connection->rheader->code == 407 && connection->proxy_type == PROXY_NORMAL && auth == FALSE) {
221                 /* authenticate with the other proxy */
222                 if (proxy_authenticate(connection)) {
223                         http_header_free(connection->rheader);
224
225                         /* only try once */
226                         auth = TRUE;
227
228                         /* authentication finished, retrieve the header again */
229                         goto auth_finished;
230                 }
231                 FREE_AND_NULL(connection->rheader->proxy_authenticate);
232
233                 /* just continue, user will see 407 warning from proxy in browser */
234         }
235
236         if (postdata != NULL)
237                 filebuf_free(postdata);
238
239         /* send template instead of web server's content when a template name matches the code */
240         snprintf(buf, sizeof(buf), "%d", connection->rheader->code);
241         ret = template_send(templates, buf, connection, connection->rheader->code);
242         if (ret == TRUE)
243                 goto out;
244
245         if (connection->rheader->location != NULL && (connection->rheader->code == 301 || connection->rheader->code == 302)) {
246                 /* redirect_do will replace the Location: header if any rules match */
247                 redirect_do(redirect_list, connection, REDIRECT_HEADER);
248
249                 /* add URL command to location: header so the feature(s) are still bypassed when
250                    browser follows redirect */
251                 if (connection->header->url_command != NULL) {
252                         ptr = url_command_add(connection, connection->rheader->location);
253                         xfree(connection->rheader->location);
254                         connection->rheader->location = ptr;
255                 }
256         }
257
258         /* no message body regardless of what header says for HEAD requests and certain return codes */
259         if (!strcasecmp(connection->header->method, "HEAD") || connection->rheader->code == 304 || connection->rheader->code == 204 || (connection->rheader->code >= 100 && connection->rheader->code < 200)) {
260                 header_send(connection->rheader, connection, CLIENT, HEADER_RESP);
261
262                 goto out;
263         }
264
265         for (url_command = connection->header->url_command; url_command && *url_command; url_command++) {
266                 if (!strcasecmp((*url_command)->command, "mime")) {
267                         /* show matching MIME filter entries, if any, for the requested URL */
268                         mime_check_show(connection);
269
270                         connection->keepalive_server = FALSE;
271
272                         goto out;
273                 }
274         }
275
276         mime_list_list = mime_check(mime_list, connection);
277         if (mime_list_list != NULL) {
278                 putlog(MMLOG_MIME, "blocked mime-type %s from %s%s", (connection->rheader->content_type != NULL) ? connection->rheader->content_type : "(none)", connection->header->host, connection->header->file);
279
280                 template_send(templates, (mime_list_list->template != NULL) ? mime_list_list->template : (mime_list->dtemplate != NULL) ? mime_list->dtemplate : "blocked", connection, 200);
281
282                 /* we have to drop the connection since we're already transferring something */
283                 connection->keepalive_server = FALSE;
284
285                 pthread_rwlock_unlock(&mime_list->lock);
286
287                 goto out;
288         }
289         pthread_rwlock_unlock(&mime_list->lock);
290
291         putlog(MMLOG_REQUEST, "%s %s:%d%s", connection->header->method, connection->header->host, connection->header->port, connection->header->file);
292
293         if (BUFFERMAX == -1 || connection->rheader->content_length <= BUFFERMAX) {
294                 connection->buffer = rewrite_do(rewrite_list, connection, NULL, REWRITE_BODY, FALSE);
295
296                 if (connection->buffer == FALSE) {
297                         external_list = external_find(external, connection);
298                         pthread_rwlock_unlock(&external->lock);
299
300                         if (external_list != NULL)
301                                 connection->buffer = TRUE;
302                 }
303
304                 if (connection->buffer == FALSE)
305                         connection->buffer = keyword_check(keyword_list, connection, NULL, FALSE);
306
307                 if (connection->buffer == FALSE) {
308                         for (url_command = connection->header->url_command; url_command && *url_command; url_command++) {
309                                 if (!strcasecmp((*url_command)->command, "score"))
310                                         connection->buffer = TRUE;
311                         }
312                 }
313         } else
314                 connection->buffer = FALSE;
315
316         if (connection->buffer == FALSE) {
317                 header_send(connection->rheader, connection, CLIENT, HEADER_RESP);
318
319                 http_transfer(connection, SERVER);
320         } else {
321                 /* this file is to be buffered and processed before sending it to the browser */
322                 filebuf = http_transfer_filebuf(connection, SERVER);
323
324                 /* decompress gzip-encoded content */
325                 if (connection->rheader->content_encoding != NULL) {
326                         if (!strcasecmp(connection->rheader->content_encoding, "gzip")) {
327                                 ret = filebuf_ungzip(filebuf);
328                                 if (ret == FALSE) {
329                                         putlog(MMLOG_ERROR, "failed to decompress gzip encoded content");
330
331                                         /* failed to decompress, just send as-is */
332                                         goto bypass;
333                                 }
334                         } else {
335                                 putlog(MMLOG_WARN, "unhandled content encoding: %s", connection->rheader->content_encoding);
336
337                                 goto bypass;
338                         }
339                 }
340
341                 /* check keyword score */
342                 ret = keyword_check(keyword_list, connection, filebuf, TRUE);
343
344                 for (url_command = connection->header->url_command; url_command && *url_command; url_command++) {
345                         if (!strcasecmp((*url_command)->command, "score")) {
346                                 score_show(connection, ret);
347
348                                 filebuf_free(filebuf);
349
350                                 goto out;
351                         }
352                 }
353
354                 if (ret) {
355                         pthread_rwlock_rdlock(&keyword_list->lock);
356                         if (ret >= keyword_list->threshold) {
357                                 putlog(MMLOG_KEYWORDS, "page above threshold with a score of %d", ret);
358
359                                 template_send(templates, (keyword_list->template != NULL) ? keyword_list->template : "blocked", connection, 200);
360
361                                 pthread_rwlock_unlock(&keyword_list->lock);
362
363                                 filebuf_free(filebuf);
364
365                                 goto out;
366                         }
367
368                         pthread_rwlock_unlock(&keyword_list->lock);
369                 }
370
371                 rewrite_do(rewrite_list, connection, filebuf, REWRITE_BODY, TRUE);
372
373                 external_list = external_find(external, connection);
374                 if (external_list != NULL && external_list->exec != NULL) {
375                         ext_filebuf = external_exec(connection, external_list->exec, filebuf, external_list->type);
376                         if (ext_filebuf != NULL) {
377                                 filebuf_free(filebuf);
378                                 filebuf = ext_filebuf;
379
380                                 if (external_list->newmime != NULL) {
381                                         FREE_AND_NULL(connection->rheader->content_type);
382
383                                         if (!strcasecmp(external_list->newmime, "STDIN")) {
384                                                 /* try extracting Content-type header from external 
385                                                    program's output */
386                                                 connection->rheader->content_type = external_getmime(filebuf);
387                                         } else
388                                                 connection->rheader->content_type = xstrdup(external_list->newmime);
389                                 }
390                         } else
391                                 putlog(MMLOG_ERROR, "failed to execute external parser '%s'", external_list->exec);
392                 }
393                 pthread_rwlock_unlock(&external->lock);
394
395                 FREE_AND_NULL(connection->rheader->content_encoding);
396                 if (connection->header->accept_encoding != NULL) {
397                         /* compress the content if the browser supports gzip encoding */
398                         ptr = strcasestr(connection->header->accept_encoding, "gzip");
399                         if (ptr != NULL) {
400                                 ret = filebuf_gzip(filebuf);
401                                 if (ret == TRUE) connection->rheader->content_encoding = xstrdup("gzip");
402                         }
403                 }
404  
405                 bypass:
406                 putlog(MMLOG_DEBUG, "filebuf size: %d", filebuf->size);
407
408                 connection->rheader->chunked = FALSE;
409                 connection->rheader->content_length = filebuf->size;
410
411                 if (connection->keepalive_client != FALSE && connection->header->keepalive != FALSE && connection->header->proxy_keepalive != FALSE) {
412                         /* we may be able to do keep-alive now since the page was buffered and the size is now known */
413                         if (connection->header->keepalive == TRUE || connection->header->proxy_keepalive == TRUE)
414                                 connection->keepalive_client = TRUE;
415                         else if (connection->header->version == HTTP_HTTP11)
416                                 connection->keepalive_client = TRUE;
417                 }
418
419                 header_send(connection->rheader, connection, CLIENT, HEADER_RESP);
420                 net_filebuf_send(filebuf, connection, CLIENT);
421
422                 filebuf_free(filebuf);
423         }
424
425       out:
426         http_header_free(connection->rheader);
427
428         return 1;
429 }
430
431 /*
432 This function handles CONNECT requests (HTTPS)
433 */
434 int protocol_connect(CONNECTION * connection)
435 {
436         pthread_mutex_lock(&threads[connection->thread].lock);
437         threads[connection->thread].flags |= THREAD_CONNECT;
438         pthread_mutex_unlock(&threads[connection->thread].lock);
439
440         putlog(MMLOG_REQUEST, "CONNECT %s:%d", connection->header->host, connection->header->port);
441
442         if (connection->proxy_type == PROXY_NORMAL)
443                 header_send(connection->header, connection, SERVER, HEADER_FORWARD);
444         else
445                 putsock(connection->client, "HTTP/1.1 Connection Established\r\n\r\n");
446
447         net_proxy(connection, -1);
448
449         return 0;
450 }
451
452 /*
453 get size of next chunk for pages using chunked encoding
454 */
455 unsigned int next_chunksize(CONNECTION * connection, int flags)
456 {
457         int x;
458         char buf[64];
459
460         x = sock_getline((flags == SERVER) ? connection->server : connection->client, buf, sizeof(buf));
461         if (x <= 0)
462                 return 0;
463
464         return strtol(buf, NULL, 16);
465 }
466
467
468 /*
469 transfer http body from one socket to another
470 */
471 void http_transfer(CONNECTION * connection, int direction)
472 {
473         int x;
474         HEADER *header;
475
476         header = (direction == SERVER) ? connection->rheader : connection->header;
477
478         if (header->content_length != -1)
479                 net_transfer(connection, direction, header->content_length);
480         else if (header->chunked)
481                 do {
482                         x = next_chunksize(connection, direction);
483                         putsock((direction == SERVER) ? connection->client : connection->server, "%x\r\n", x);
484                         /* rfc says chunks must end in \r\n */
485                         net_transfer(connection, direction, x + 2);
486                 } while (x > 0);
487         else
488                 net_transfer(connection, direction, -1);
489
490 }
491
492 /*
493 transfer http body into filebuf
494 */
495 FILEBUF *http_transfer_filebuf(CONNECTION * connection, int direction)
496 {
497         int x;
498         FILEBUF *filebuf;
499         HEADER *header;
500         SOCKET *sock;
501
502         filebuf = filebuf_new();
503
504         header = (direction == SERVER) ? connection->rheader : connection->header;
505         sock = (direction == SERVER) ? connection->server : connection->client;
506
507         if (header->content_length != -1)
508                 net_filebuf_read(filebuf, connection, direction, header->content_length);
509         else if (header->chunked)
510                 do {
511                         x = next_chunksize(connection, direction);
512                         net_filebuf_read(filebuf, connection, direction, x);
513                         /* discard the \r\n from the server */
514                         sock_getline(sock, NULL, -1);
515                 } while (x > 0);
516         else
517                 net_filebuf_read(filebuf, connection, direction, -1);
518
519         return filebuf;
520 }
521
522 /* 
523 read the body and discard it 
524 */
525 void http_transfer_discard(CONNECTION * connection, int direction)
526 {
527         int x;
528         HEADER *header;
529         SOCKET *sock;
530
531         header = (direction == SERVER) ? connection->rheader : connection->header;
532         sock = (direction == SERVER) ? connection->server : connection->client;
533
534         if (header->content_length != -1)
535                 net_filebuf_read(NULL, connection, direction, header->content_length);
536         else if (header->chunked)
537                 do {
538                         x = next_chunksize(connection, direction);
539                         net_filebuf_read(NULL, connection, direction, x);
540                         /* discard the \r\n from the server */
541                         sock_getline(sock, NULL, -1);
542                 } while (x > 0);
543         else
544                 net_filebuf_read(NULL, connection, direction, -1);
545
546 }
547
548 int proxy_authenticate(CONNECTION * connection)
549 {
550         if (connection->rheader->proxy_authenticate == NULL)
551                 return FALSE;
552
553         if (!strcasecmp(connection->rheader->proxy_authenticate, "NTLM")) {
554                 /* NTLM authentication */
555                 return send_ntlm_response(connection);
556         } else if (!strncasecmp(connection->rheader->proxy_authenticate, "Basic", 5)) {
557                 /* Basic authentication */
558                 return send_basic_response(connection);
559         }
560
561         return FALSE;
562 }
563
564 /* perform an NTLM handshake, the sequence of events looks like this:
565     1: C -->  S   GET ...
566
567     2: C <--  S   401 Unauthorized
568                   Proxy-Authenticate: NTLM
569     
570     3: C  --> S   GET ...
571                   Proxy-Authorization: NTLM <base64-encoded type-1-message>
572     
573     4: C <--  S   401 Unauthorized
574                   Proxy-Authenticate: NTLM <base64-encoded type-2-message>
575     
576     5: C  --> S   GET ...
577                   Proxy-Authorization: NTLM <base64-encoded type-3-message>
578     
579     6: C <--  S   200 Ok
580 */
581 int send_ntlm_response(CONNECTION * connection)
582 {
583         int ret, oldkeep;
584         unsigned char buf[4096], buf2[4096], *headbuf;
585         HEADER *header;
586
587         /* client must do keep-alive during authentication */
588         oldkeep = connection->keepalive_server;
589         connection->keepalive_server = TRUE;
590
591         /* don't need the body of the 407 message */
592         http_transfer_discard(connection, SERVER);
593
594         /* send type-1 message */
595         /* note: buildSmbNtml* knows how to deal with NULL arguments */
596         buildSmbNtlmAuthRequest((tSmbNtlmAuthRequest *) buf, connection->proxy_username, connection->proxy_domain);
597         to64frombits(buf2, buf, SmbLength((tSmbNtlmAuthResponse *) buf));
598
599         FREE_AND_NULL(connection->proxy_auth);
600         snprintf(buf, sizeof(buf), "NTLM %s", buf2);
601         connection->proxy_auth = xstrdup(buf);
602
603         header_send(connection->header, connection, SERVER, HEADER_FORWARD);
604
605         headbuf = header_get(connection, SERVER, TIMEOUT);
606         if (headbuf == NULL) {
607                 /* squid and maybe some other proxies will disconnect after sending
608                    the first 407 message regardless of Connection header */
609                 ret = protocol_reconnect(connection);
610                 if (ret == -1)
611                         goto error;
612
613                 header_send(connection->header, connection, SERVER, HEADER_FORWARD);
614
615                 headbuf = header_get(connection, SERVER, TIMEOUT);
616                 if (headbuf == NULL)
617                         goto error;
618         }
619
620         header = http_header_parse_response(headbuf);
621         xfree(headbuf);
622         if (header == NULL)
623                 goto error;
624
625         http_header_free(connection->rheader);
626         connection->rheader = header;
627
628         if (header->proxy_authenticate == NULL || strncasecmp(header->proxy_authenticate, "NTLM ", 5))
629                 goto error;
630
631         http_transfer_discard(connection, SERVER);
632
633         /* parse type-2 message */
634         from64tobits(buf, &header->proxy_authenticate[5]);
635         buildSmbNtlmAuthResponse((tSmbNtlmAuthChallenge *) buf, (tSmbNtlmAuthResponse *) buf2, connection->proxy_username, connection->proxy_password);
636
637         /* send type-3 message */
638         to64frombits(buf, buf2, SmbLength((tSmbNtlmAuthResponse *) buf2));
639         snprintf(buf2, sizeof(buf2), "NTLM %s", buf);
640
641         FREE_AND_NULL(connection->proxy_auth);
642         connection->proxy_auth = xstrdup(buf2);
643
644         header_send(connection->header, connection, SERVER, HEADER_FORWARD);
645
646         connection->keepalive_server = oldkeep;
647
648         return TRUE;
649
650       error:
651         connection->keepalive_server = oldkeep;
652
653         return FALSE;
654 }
655
656 /* basic proxy authentication */
657 int send_basic_response(CONNECTION * connection)
658 {
659         char buf[4096], buf2[4096];
660
661         /* discard body of 407 authentication required message */
662         http_transfer_discard(connection, SERVER);
663
664         snprintf(buf, sizeof(buf), "%s:%s", (connection->proxy_username != NULL) ? connection->proxy_username : "", (connection->proxy_password != NULL) ? connection->proxy_password : "");
665         to64frombits(buf2, buf, strlen(buf));
666
667         snprintf(buf, sizeof(buf), "Basic %s", buf2);
668
669         FREE_AND_NULL(connection->proxy_auth);
670         connection->proxy_auth = xstrdup(buf);
671
672         header_send(connection->header, connection, SERVER, HEADER_FORWARD);
673
674         return TRUE;
675 }