/* Internal "http" protocol implementation */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef HAVE_LIMITS_H
#include <limits.h>
#endif

#include "elinks.h"

#include "cache/cache.h"
#include "config/options.h"
#include "cookies/cookies.h"
#include "intl/charsets.h"
#include "intl/gettext/libintl.h"
#include "main/module.h"
#include "network/connection.h"
#include "network/progress.h"
#include "network/socket.h"
#include "osdep/ascii.h"
#include "osdep/osdep.h"
#include "osdep/sysname.h"
#include "protocol/auth/auth.h"
#include "protocol/auth/digest.h"
#include "protocol/date.h"
#include "protocol/header.h"
#include "protocol/http/blacklist.h"
#include "protocol/http/codes.h"
#include "protocol/http/http.h"
#include "protocol/uri.h"
#include "session/session.h"
#include "terminal/terminal.h"
#include "util/base64.h"
#include "util/conv.h"
#include "util/memory.h"
#include "util/string.h"

#ifdef CONFIG_GSSAPI
#include "http_negotiate.h"
#endif

/* These macros concern the struct http_version defined in the http.h */
#define HTTP_0_9(x)	 ((x).major == 0 && (x).minor == 9)
#define HTTP_1_0(x)	 ((x).major == 1 && (x).minor == 0)
#define HTTP_1_1(x)	 ((x).major == 1 && (x).minor == 1)
#define PRE_HTTP_1_0(x)  ((x).major < 1)
#define PRE_HTTP_1_1(x)  (PRE_HTTP_1_0(x) || HTTP_1_0(x))
#define POST_HTTP_1_0(x) ((x).major > 1 || ((x).major == 1 && (x).minor > 0))
#define POST_HTTP_1_1(x) ((x).major > 1 || ((x).major == 1 && (x).minor > 1))


#define LEN_CHUNKED -2 /* == we get data in unknown number of chunks */
#define LEN_FINISHED 0

	/* Either bytes coming in this chunk yet or "parser state". */
#define CHUNK_DATA_END	-3
#define CHUNK_ZERO_SIZE	-2
#define CHUNK_SIZE	-1

static struct auth_entry proxy_auth;

static unsigned char *accept_charset = NULL;


static union option_info http_options[] = {
	INIT_OPT_TREE("protocol", N_("HTTP"),
		"http", 0,
		N_("HTTP-specific options.")),


	INIT_OPT_TREE("protocol.http", N_("Server bug workarounds"),
		"bugs", 0,
		N_("Server-side HTTP bugs workarounds.")),

	INIT_OPT_BOOL("protocol.http.bugs", N_("Do not send Accept-Charset"),
		"accept_charset", 0, 1,
		N_("The Accept-Charset header is quite long and sending it "
		"can trigger bugs in some rarely found servers.")),

	INIT_OPT_BOOL("protocol.http.bugs", N_("Allow blacklisting"),
		"allow_blacklist", 0, 1,
		N_("Allow blacklisting of buggy servers.")),

	INIT_OPT_BOOL("protocol.http.bugs", N_("Broken 302 redirects"),
		"broken_302_redirect", 0, 1,
		N_("Broken 302 redirect (violates RFC but compatible with "
		"Netscape). This is a problem for a lot of web discussion "
		"boards and the like. If they will do strange things to you, "
		"try to play with this.")),

	INIT_OPT_BOOL("protocol.http.bugs", N_("No keepalive after POST requests"),
		"post_no_keepalive", 0, 0,
		N_("Disable keepalive connection after POST request.")),

	INIT_OPT_BOOL("protocol.http.bugs", N_("Use HTTP/1.0"),
		"http10", 0, 0,
		N_("Use HTTP/1.0 protocol instead of HTTP/1.1.")),

	INIT_OPT_TREE("protocol.http", N_("Proxy configuration"),
		"proxy", 0,
		N_("HTTP proxy configuration.")),

	INIT_OPT_STRING("protocol.http.proxy", N_("Host and port-number"),
		"host", 0, "",
		N_("Host and port-number (host:port) of the HTTP proxy, "
		"or blank. If it's blank, HTTP_PROXY environment variable "
		"is checked as well.")),

	INIT_OPT_STRING("protocol.http.proxy", N_("Username"),
		"user", 0, "",
		N_("Proxy authentication username.")),

	INIT_OPT_STRING("protocol.http.proxy", N_("Password"),
		"passwd", 0, "",
		N_("Proxy authentication password.")),


	INIT_OPT_TREE("protocol.http", N_("Referer sending"),
		"referer", 0,
		N_("HTTP referer sending options. HTTP referer is a special "
		"header sent in the HTTP requests, which is supposed to "
		"contain the previous page visited by the browser."
		"This way, the server can know what link did you follow "
		"when accessing that page. However, this behaviour can "
		"unfortunately considerably affect privacy and can lead even "
		"to a security problem on some badly designed web pages.")),

	INIT_OPT_INT("protocol.http.referer", N_("Policy"),
		"policy", 0,
		REFERER_NONE, REFERER_TRUE, REFERER_TRUE,
		N_("Mode of sending HTTP referer:\n"
		"0 is send no referer\n"
		"1 is send current URL as referer\n"
		"2 is send fixed fake referer\n"
		"3 is send previous URL as referer (correct, but insecure)")),

	INIT_OPT_STRING("protocol.http.referer", N_("Fake referer URL"),
		"fake", 0, "",
		N_("Fake referer to be sent when policy is 2.")),


	INIT_OPT_STRING("protocol.http", N_("Send Accept-Language header"),
		"accept_language", 0, "",
		N_("Send Accept-Language header.")),

	INIT_OPT_BOOL("protocol.http", N_("Use UI language as Accept-Language"),
		"accept_ui_language", 0, 1,
		N_("Request localised versions of documents from web-servers "
		"(using the Accept-Language header) using the language "
		"you have configured for ELinks' user-interface (this also "
		"affects navigator.language ECMAScript value available to "
		"scripts). Note that some see this as a potential security "
		"risk because it tells web-masters and the FBI sniffers "
		"about your language preference.")),

	/* http://www.eweek.com/c/a/Desktops-and-Notebooks/Intel-Psion-End-Dispute-Concerning-Netbook-Trademark-288875/
	 * responds with "Transfer-Encoding: chunked" and
	 * "Content-Encoding: gzip" but does not compress the first chunk
	 * and the last chunk, causing ELinks to display garbage.
	 * (If User-Agent includes "Gecko" (case sensitive), then
	 * that server correctly compresses the whole stream.)
	 * ELinks should instead report the decompression error (bug 1017)
	 * or perhaps even blacklist the server for compression and retry.
	 * Until that has been implemented, disable compression by default.  */
	INIT_OPT_BOOL("protocol.http", N_("Enable on-the-fly compression"),
		"compression", 0, 0,
		N_("If enabled, the capability to receive compressed content "
		"(gzip and/or bzip2) is announced to the server, which "
		"usually sends the reply compressed, thus saving some "
		"bandwidth at slight CPU expense.\n"
		"\n"
		"If ELinks displays a incomplete page or garbage, try "
		"disabling this option. If that helps, there may be a bug in "
		"the decompression part of ELinks. Please report such bugs.\n"
		"\n"
		"If ELinks has been compiled without compression support, "
		"this option has no effect. To check the supported features, "
		"see Help -> About.")),

	INIT_OPT_BOOL("protocol.http", N_("Activate HTTP TRACE debugging"),
		"trace", 0, 0,
		N_("If active, all HTTP requests are sent with TRACE as "
		"their method rather than GET or POST. This is useful for "
		"debugging of both ELinks and various server-side scripts "
		"--- the server only returns the client's request back to "
		"the client verbatim. Note that this type of request may "
		"not be enabled on all servers.")),

	/* OSNews.com is supposed to be relying on the textmode token, at least. */
	INIT_OPT_STRING("protocol.http", N_("User-agent identification"),
		"user_agent", 0, "ELinks/%v (textmode; %s; %t-%b)",
		N_("Change the User Agent ID. That means identification "
		"string, which is sent to HTTP server when a document is "
		"requested. The 'textmode' token in the first field is our "
		"silent attempt to establish this as a standard for new "
		"textmode user agents, so that the webmasters can have "
		"just a single uniform test for these if they are e.g. "
		"pushing some lite version to them automagically.\n"
		"\n"
		"Use \" \" if you don't want any User-Agent header to be sent "
		"at all. URI rewriting rules may still include parameters "
		"that reveal you are using ELinks.\n"
		"\n"
		"%v in the string means ELinks version,\n"
		"%s in the string means system identification,\n"
		"%t in the string means size of the terminal,\n"
		"%b in the string means number of bars displayed by ELinks.")),


	INIT_OPT_TREE("protocol", N_("HTTPS"),
  		"https", 0,
		N_("HTTPS-specific options.")),

	INIT_OPT_TREE("protocol.https", N_("Proxy configuration"),
	  	"proxy", 0,
		N_("HTTPS proxy configuration.")),

	INIT_OPT_STRING("protocol.https.proxy", N_("Host and port-number"),
	  	"host", 0, "",
		N_("Host and port-number (host:port) of the HTTPS CONNECT "
		"proxy, or blank. If it's blank, HTTPS_PROXY environment "
		"variable is checked as well.")),
	NULL_OPTION_INFO,
};

static void done_http();

struct module http_protocol_module = struct_module(
	/* name: */		N_("HTTP"),
	/* options: */		http_options,
	/* hooks: */		NULL,
	/* submodules: */	NULL,
	/* data: */		NULL,
	/* init: */		NULL,
	/* done: */		done_http
);


static void
done_http(void)
{
	mem_free_if(proxy_auth.realm);
	mem_free_if(proxy_auth.nonce);
	mem_free_if(proxy_auth.opaque);

	free_blacklist();

	if (accept_charset)
		mem_free(accept_charset);
}

static void
init_accept_charset(void)
{
	struct string ac;
	unsigned char *cs;
	int i;

	if (!init_string(&ac)) return;

	for (i = 0; (cs = get_cp_mime_name(i)); i++) {
		if (ac.length) {
			add_to_string(&ac, ", ");
		} else {
			add_to_string(&ac, "Accept-Charset: ");
		}
		add_to_string(&ac, cs);
	}

	if (ac.length) {
		add_crlf_to_string(&ac);
	}

	accept_charset = squeezastring(&ac);

	done_string(&ac);
}


unsigned char *
subst_user_agent(unsigned char *fmt, unsigned char *version,
		 unsigned char *sysname, unsigned char *termsize)
{
	struct string agent;

	if (!init_string(&agent)) return NULL;

	while (*fmt) {
		int p;

		for (p = 0; fmt[p] && fmt[p] != '%'; p++);

		add_bytes_to_string(&agent, fmt, p);
		fmt += p;

		if (*fmt != '%') continue;

		fmt++;
		switch (*fmt) {
			case 'b':
				if (!list_empty(sessions)) {
					unsigned char bs[4] = "";
					int blen = 0;
					struct session *ses = sessions.prev;
					int bars = ses->status.show_status_bar
						+ ses->status.show_tabs_bar
						+ ses->status.show_title_bar;

					ulongcat(bs, &blen, bars, 2, 0);
					add_to_string(&agent, bs);
				}
				break;
			case 'v':
				add_to_string(&agent, version);
				break;
			case 's':
				add_to_string(&agent, sysname);
				break;
			case 't':
				if (termsize)
					add_to_string(&agent, termsize);
				break;
			default:
				add_bytes_to_string(&agent, fmt - 1, 2);
				break;
		}
		if (*fmt) fmt++;
	}

	return agent.source;
}

static void
add_url_to_http_string(struct string *header, struct uri *uri, int components)
{
	/* This block substitues spaces in URL by %20s. This is
	 * certainly not the right place where to do it, but now the
	 * behaviour is at least improved compared to what we had
	 * before. We should probably encode all URLs as early as
	 * possible, and possibly decode them back in protocol
	 * backends. --pasky */
	unsigned char *string = get_uri_string(uri, components);
	unsigned char *data = string;

	if (!string) return;

	while (*data) {
		int len = strcspn(data, " \t\r\n\\");

		add_bytes_to_string(header, data, len);

		if (!data[len]) break;

		if (data[len++] == '\\')
			add_char_to_string(header, '/');
		else
			add_to_string(header, "%20");

		data	+= len;
	}

	mem_free(string);
}

/* Parse from @end - 1 to @start and set *@value to integer found.
 * It returns -1 if not a number, 0 otherwise.
 * @end should be > @start. */
static int
revstr2num(unsigned char *start, unsigned char *end, int *value)
{
	int q = 1, val = 0;

	do {
		--end;
		if (!isdigit(*end)) return -1; /* NaN */
		val += (*end - '0') * q;
		q *= 10;
	} while (end > start);

	*value = val;
	return 0;
}

/* This function extracts code, major and minor version from string
 * "\s*HTTP/\d+.\d+\s+\d\d\d..."
 * It returns a negative value on error, 0 on success.
 */
static int
get_http_code(struct read_buffer *rb, int *code, struct http_version *version)
{
	unsigned char *head = rb->data;
	unsigned char *start;

	*code = 0;
	version->major = 0;
	version->minor = 0;

	/* Ignore spaces. */
	while (*head == ' ') head++;

	/* HTTP/ */
	if (c_toupper(*head) != 'H' || c_toupper(*++head) != 'T' ||
	    c_toupper(*++head) != 'T' || c_toupper(*++head) != 'P'
	    || *++head != '/')
		return -1;

	/* Version */
	start = ++head;
	/* Find next '.' */
	while (*head && *head != '.') head++;
	/* Sanity check. */
	if (!*head || !(head - start)
	    || (head - start) > 4
	    || !isdigit(*(head + 1)))
		return -2;

	/* Extract major version number. */
	if (revstr2num(start, head, &version->major)) return -3; /* NaN */

	start = head + 1;

	/* Find next ' '. */
	while (*head && *head != ' ') head++;
	/* Sanity check. */
	if (!*head || !(head - start) || (head - start) > 4) return -4;

	/* Extract minor version number. */
	if (revstr2num(start, head, &version->minor)) return -5; /* NaN */

	/* Ignore spaces. */
	while (*head == ' ') head++;

	/* Sanity check for code. */
	if (head[0] < '1' || head[0] > '9' ||
	    !isdigit(head[1]) ||
	    !isdigit(head[2]))
		return -6; /* Invalid code. */

	/* Extract code. */
	*code = (head[0] - '0') * 100 + (head[1] - '0') * 10 + head[2] - '0';

	return 0;
}

static int
check_http_server_bugs(struct uri *uri, struct http_connection_info *http,
		       unsigned char *head)
{
	unsigned char *server;
	const unsigned char *const *s;
	static const unsigned char *const buggy_servers[] = {
		"mod_czech/3.1.0",
		"Purveyor",
		"Netscape-Enterprise",
		NULL
	};

	if (!get_opt_bool("protocol.http.bugs.allow_blacklist", NULL)
	    || HTTP_1_0(http->sent_version))
		return 0;

	server = parse_header(head, "Server", NULL);
	if (!server)
		return 0;

	for (s = buggy_servers; *s; s++) {
		if (strstr((const char *)server, *s)) {
			add_blacklist_entry(uri, SERVER_BLACKLIST_HTTP10);
			break;
		}
	}

	mem_free(server);
	return (*s != NULL);
}

static void
http_end_request(struct connection *conn, struct connection_state state,
		 int notrunc)
{
	struct http_connection_info *http;

	shutdown_connection_stream(conn);

	/* shutdown_connection_stream() should not change conn->info,
	 * but in case it does, read conn->info only after the call.  */
	http = conn->info;
	if (http)
		done_http_post(&http->post);

	if (http && !http->close
	    && (!conn->socket->ssl) /* We won't keep alive ssl connections */
	    && (!get_opt_bool("protocol.http.bugs.post_no_keepalive", NULL)
		|| !conn->uri->post)) {
		if (is_in_state(state, S_OK) && conn->cached)
			normalize_cache_entry(conn->cached, !notrunc ? conn->from : -1);
		set_connection_state(conn, state);
		add_keepalive_connection(conn, HTTP_KEEPALIVE_TIMEOUT, NULL);
	} else {
		abort_connection(conn, state);
	}
}

static void http_send_header(struct socket *);

void
http_protocol_handler(struct connection *conn)
{
	/* setcstate(conn, S_CONN); */

	if (!has_keepalive_connection(conn)) {
		make_connection(conn->socket, conn->uri, http_send_header,
				conn->cache_mode >= CACHE_MODE_FORCE_RELOAD);
	} else {
		http_send_header(conn->socket);
	}
}

void
proxy_protocol_handler(struct connection *conn)
{
	http_protocol_handler(conn);
}

#define IS_PROXY_URI(x) ((x)->protocol == PROTOCOL_PROXY)

#define connection_is_https_proxy(conn) \
	(IS_PROXY_URI((conn)->uri) && (conn)->proxied_uri->protocol == PROTOCOL_HTTPS)

/** connection.done points to this function if connection.info points
 * to a struct http_connection_info.  */
static void
done_http_connection(struct connection *conn)
{
	struct http_connection_info *http = conn->info;

	done_http_post(&http->post);
	mem_free(http);
	conn->info = NULL;
	conn->done = NULL;
}

struct http_connection_info *
init_http_connection_info(struct connection *conn, int major, int minor, int close)
{
	struct http_connection_info *http;

	http = mem_calloc(1, sizeof(*http));
	if (!http) {
		http_end_request(conn, connection_state(S_OUT_OF_MEM), 0);
		return NULL;
	}

	http->sent_version.major = major;
	http->sent_version.minor = minor;
	http->close = close;

	init_http_post(&http->post);

	/* The CGI code uses this too and blacklisting expects a host name. */
	if (conn->proxied_uri->protocol != PROTOCOL_FILE)
		http->bl_flags = get_blacklist_flags(conn->proxied_uri);

	if (http->bl_flags & SERVER_BLACKLIST_HTTP10
	    || get_opt_bool("protocol.http.bugs.http10", NULL)) {
		http->sent_version.major = 1;
		http->sent_version.minor = 0;
	}

	/* If called from HTTPS proxy connection the connection info might have
	 * already been allocated. */
	if (conn->done) {
		conn->done(conn);
		conn->done = NULL;
	}
	mem_free_set(&conn->info, http);
	conn->done = done_http_connection;

	return http;
}

static void
accept_encoding_header(struct string *header)
{
#if defined(CONFIG_GZIP) || defined(CONFIG_BZIP2) || defined(CONFIG_LZMA) || defined(CONFIG_BROTLI) || defined(CONFIG_ZSTD)
	int comma = 0;

	add_to_string(header, "Accept-Encoding: ");

#ifdef CONFIG_ZSTD
	add_to_string(header, "zstd");
	comma = 1;
#endif

#ifdef CONFIG_BROTLI
	if (comma) add_to_string(header, ", ");
	add_to_string(header, "br");
	comma = 1;
#endif

#ifdef CONFIG_BZIP2
	if (comma) add_to_string(header, ", ");
	add_to_string(header, "bzip2");
	comma = 1;
#endif

#ifdef CONFIG_GZIP
	if (comma) add_to_string(header, ", ");
	add_to_string(header, "gzip");
	comma = 1;
#endif

#ifdef CONFIG_LZMA
	if (comma) add_to_string(header, ", ");
	add_to_string(header, "lzma");
#endif
	add_crlf_to_string(header);
#endif
}

#define POST_BUFFER_SIZE 16384

static void
send_more_post_data(struct socket *socket)
{
	struct connection *conn = socket->conn;
	struct http_connection_info *http = conn->info;
	unsigned char buffer[POST_BUFFER_SIZE];
	int got;
	struct connection_state error;

	got = read_http_post(&http->post, buffer, POST_BUFFER_SIZE, &error);
	if (got < 0) {
		http_end_request(conn, error, 0);
	} else if (got > 0) {
		write_to_socket(socket, buffer, got, connection_state(S_TRANS),
				send_more_post_data);
	} else {		/* got == 0, meaning end of data */
		/* Can't use request_from_socket() because there's no
		 * more data to write.  */
		struct read_buffer *rb = alloc_read_buffer(socket);

		socket->state = SOCKET_END_ONCLOSE;
		if (rb)
			read_from_socket(socket, rb, connection_state(S_SENT),
					 http_got_header);
		else
			http_end_request(conn, connection_state(S_OUT_OF_MEM),
					 0);
	}
}



static void
http_send_header(struct socket *socket)
{
	struct connection *conn = socket->conn;
	struct http_connection_info *http;
	int trace = get_opt_bool("protocol.http.trace", NULL);
	struct string header;
	unsigned char *post_data = NULL;
	struct auth_entry *entry = NULL;
	struct uri *uri = conn->proxied_uri; /* Set to the real uri */
	unsigned char *optstr;
	int use_connect, talking_to_proxy;

	/* Sanity check for a host */
	if (!uri || !uri->host || !*uri->host || !uri->hostlen) {
		http_end_request(conn, connection_state(S_BAD_URL), 0);
		return;
	}

	http = init_http_connection_info(conn, 1, 1, 0);
	if (!http) return;

	if (!init_string(&header)) {
		http_end_request(conn, connection_state(S_OUT_OF_MEM), 0);
		return;
	}

	if (!conn->cached) conn->cached = find_in_cache(uri);

	talking_to_proxy = IS_PROXY_URI(conn->uri) && !conn->socket->ssl;
	use_connect = connection_is_https_proxy(conn) && !conn->socket->ssl;

	if (trace) {
		add_to_string(&header, "TRACE ");
	} else if (use_connect) {
		add_to_string(&header, "CONNECT ");
		/* In CONNECT requests, we send only a subset of the
		 * headers to the proxy.  See the "CONNECT:" comments
		 * below.  After the CONNECT request succeeds, we
		 * negotiate TLS with the real server and make a new
		 * HTTP request that includes all the headers.  */
	} else if (uri->post) {
		add_to_string(&header, "POST ");
		conn->unrestartable = 1;
	} else {
		add_to_string(&header, "GET ");
	}

	if (!talking_to_proxy) {
		add_char_to_string(&header, '/');
	}

	if (use_connect) {
		/* Add port if it was specified or the default port */
		add_uri_to_string(&header, uri, URI_HTTP_CONNECT);
	} else {
		if (connection_is_https_proxy(conn) && conn->socket->ssl) {
			add_url_to_http_string(&header, uri, URI_DATA);

		} else if (talking_to_proxy) {
			add_url_to_http_string(&header, uri, URI_PROXY);

		} else {
			add_url_to_http_string(&header, conn->uri, URI_DATA);
		}
	}

	add_to_string(&header, " HTTP/");
	add_long_to_string(&header, http->sent_version.major);
	add_char_to_string(&header, '.');
	add_long_to_string(&header, http->sent_version.minor);
	add_crlf_to_string(&header);

	/* CONNECT: Sending a Host header seems pointless as the same
	 * information is already in the CONNECT line.  It's harmless
	 * though and Mozilla does it too.  */
	add_to_string(&header, "Host: ");
	add_uri_to_string(&header, uri, URI_HTTP_HOST);
	add_crlf_to_string(&header);

	/* CONNECT: Proxy-Authorization is intended to be seen by the proxy.  */
	if (talking_to_proxy) {
		unsigned char *user = get_opt_str("protocol.http.proxy.user", NULL);
		unsigned char *passwd = get_opt_str("protocol.http.proxy.passwd", NULL);

		if (proxy_auth.digest) {
			unsigned char *response;
			int userlen = int_min(strlen(user), AUTH_USER_MAXLEN - 1);
			int passwordlen = int_min(strlen(passwd), AUTH_PASSWORD_MAXLEN - 1);

			if (userlen)
				memcpy(proxy_auth.user, user, userlen);
			proxy_auth.user[userlen] = '\0';
			if (passwordlen)
				memcpy(proxy_auth.password, passwd, passwordlen);
			proxy_auth.password[passwordlen] = '\0';

			/* FIXME: @uri is the proxied URI. Maybe the passed URI
			 * should be the proxy URI aka conn->uri. --jonas */
			response = get_http_auth_digest_response(&proxy_auth, uri);
			if (response) {
				add_to_string(&header, "Proxy-Authorization: Digest ");
				add_to_string(&header, response);
				add_crlf_to_string(&header);

				mem_free(response);
			}

		} else {
			if (user[0]) {
				unsigned char *proxy_data;

				proxy_data = straconcat(user, ":", passwd, (unsigned char *) NULL);
				if (proxy_data) {
					unsigned char *proxy_64 = base64_encode(proxy_data);

					if (proxy_64) {
						add_to_string(&header, "Proxy-Authorization: Basic ");
						add_to_string(&header, proxy_64);
						add_crlf_to_string(&header);
						mem_free(proxy_64);
					}
					mem_free(proxy_data);
				}
			}
		}
	}

	/* CONNECT: User-Agent does not reveal anything about the
	 * resource we're fetching, and it may help the proxy return
	 * better error messages.  */
	optstr = get_opt_str("protocol.http.user_agent", NULL);
	if (*optstr && strcmp(optstr, " ")) {
		unsigned char *ustr, ts[64] = "";
		/* TODO: Somehow get the terminal in which the
		 * document will actually be displayed.  */
		struct terminal *term = get_default_terminal();

		add_to_string(&header, "User-Agent: ");

		if (term) {
			unsigned int tslen = 0;

			ulongcat(ts, &tslen, term->width, 3, 0);
			ts[tslen++] = 'x';
			ulongcat(ts, &tslen, term->height, 3, 0);
		}
		ustr = subst_user_agent(optstr, VERSION_STRING, system_name,
					ts);

		if (ustr) {
			add_to_string(&header, ustr);
			mem_free(ustr);
		}

		add_crlf_to_string(&header);
	}

	/* CONNECT: Referer probably is a secret page in the HTTPS
	 * server, so don't reveal it to the proxy.  */
	if (!use_connect) {
		switch (get_opt_int("protocol.http.referer.policy", NULL)) {
			case REFERER_NONE:
				/* oh well */
				break;

			case REFERER_FAKE:
				optstr = get_opt_str("protocol.http.referer.fake", NULL);
				if (!optstr[0]) break;
				add_to_string(&header, "Referer: ");
				add_to_string(&header, optstr);
				add_crlf_to_string(&header);
				break;

			case REFERER_TRUE:
				if (!conn->referrer) break;
				add_to_string(&header, "Referer: ");
				add_url_to_http_string(&header, conn->referrer, URI_HTTP_REFERRER);
				add_crlf_to_string(&header);
				break;

			case REFERER_SAME_URL:
				add_to_string(&header, "Referer: ");
				add_url_to_http_string(&header, uri, URI_HTTP_REFERRER);
				add_crlf_to_string(&header);
				break;
		}
	}

	/* CONNECT: Do send all Accept* headers to the CONNECT proxy,
	 * because they do not reveal anything about the resource
	 * we're going to request via TLS, and they may affect the
	 * error message if the CONNECT request fails.
	 *
	 * If ELinks is ever changed to vary its Accept headers based
	 * on what it intends to do with the returned resource, e.g.
	 * sending "Accept: text/css" when it wants an external
	 * stylesheet, then it should do that only in the inner GET
	 * and not in the outer CONNECT.  */
	add_to_string(&header, "Accept: */*");
	add_crlf_to_string(&header);

	if (get_opt_bool("protocol.http.compression", NULL))
		accept_encoding_header(&header);

	if (!accept_charset) {
		init_accept_charset();
	}

	if (!(http->bl_flags & SERVER_BLACKLIST_NO_CHARSET)
	    && !get_opt_bool("protocol.http.bugs.accept_charset", NULL)
	    && accept_charset) {
		add_to_string(&header, accept_charset);
	}

	optstr = get_opt_str("protocol.http.accept_language", NULL);
	if (optstr[0]) {
		add_to_string(&header, "Accept-Language: ");
		add_to_string(&header, optstr);
		add_crlf_to_string(&header);
	}
#ifdef CONFIG_NLS
	else if (get_opt_bool("protocol.http.accept_ui_language", NULL)) {
		unsigned char *code = language_to_iso639(current_language);

		if (code) {
			add_to_string(&header, "Accept-Language: ");
			add_to_string(&header, code);
			add_crlf_to_string(&header);
		}
	}
#endif

	/* CONNECT: Proxy-Connection is intended to be seen by the
	 * proxy.  If the CONNECT request succeeds, then the proxy
	 * will forward the remainder of the TCP connection to the
	 * origin server, and Proxy-Connection does not matter; but
	 * if the request fails, then Proxy-Connection may matter.  */
	/* FIXME: What about post-HTTP/1.1?? --Zas */
	if (HTTP_1_1(http->sent_version)) {
		if (!IS_PROXY_URI(conn->uri)) {
			add_to_string(&header, "Connection: ");
		} else {
			add_to_string(&header, "Proxy-Connection: ");
		}

		if (!uri->post || !get_opt_bool("protocol.http.bugs.post_no_keepalive", NULL)) {
			add_to_string(&header, "Keep-Alive");
		} else {
			add_to_string(&header, "close");
		}
		add_crlf_to_string(&header);
	}

	/* CONNECT: Do not tell the proxy anything we have cached
	 * about the resource.  */
	if (!use_connect && conn->cached) {
		if (!conn->cached->incomplete && conn->cached->head
		    && conn->cache_mode <= CACHE_MODE_CHECK_IF_MODIFIED) {
			if (conn->cached->last_modified) {
				add_to_string(&header, "If-Modified-Since: ");
				add_to_string(&header, conn->cached->last_modified);
				add_crlf_to_string(&header);
			}
			if (conn->cached->etag) {
				add_to_string(&header, "If-None-Match: ");
				add_to_string(&header, conn->cached->etag);
				add_crlf_to_string(&header);
			}
		}
	}

	/* CONNECT: Let's send cache control headers to the proxy too;
	 * they may affect DNS caching.  */
	if (conn->cache_mode >= CACHE_MODE_FORCE_RELOAD) {
		add_to_string(&header, "Pragma: no-cache");
		add_crlf_to_string(&header);
		add_to_string(&header, "Cache-Control: no-cache");
		add_crlf_to_string(&header);
	}

	/* CONNECT: Do not reveal byte ranges to the proxy.  It can't
	 * do anything good with that information anyway.  */
	if (!use_connect && (conn->from || conn->progress->start > 0)) {
		/* conn->from takes precedence. conn->progress.start is set only the first
		 * time, then conn->from gets updated and in case of any retries
		 * etc we have everything interesting in conn->from already. */
		add_to_string(&header, "Range: bytes=");
		add_long_to_string(&header, conn->from ? conn->from : conn->progress->start);
		add_char_to_string(&header, '-');
		add_crlf_to_string(&header);
	}

	/* CONNECT: The Authorization header is for the origin server only.  */
	if (!use_connect) {
#ifdef CONFIG_GSSAPI
		if (http_negotiate_output(uri, &header) != 0)
#endif
			entry = find_auth(uri);
	}

	if (entry) {
		if (entry->digest) {
			unsigned char *response;

			response = get_http_auth_digest_response(entry, uri);
			if (response) {
				add_to_string(&header, "Authorization: Digest ");
				add_to_string(&header, response);
				add_crlf_to_string(&header);

				mem_free(response);
			}

		} else {
			/* RFC2617 section 2 [Basic Authentication Scheme]
			 *
			 * To receive authorization, the client sends the userid
			 * and password, separated by a single colon (":")
			 * character, within a base64 [7] encoded string in the
			 * credentials. */
			unsigned char *id;

			/* Create base64 encoded string. */
			id = straconcat(entry->user, ":", entry->password,
					(unsigned char *) NULL);
			if (id) {
				unsigned char *base64 = base64_encode(id);

				mem_free_set(&id, base64);
			}

			if (id) {
				add_to_string(&header, "Authorization: Basic ");
				add_to_string(&header, id);
				add_crlf_to_string(&header);
				mem_free(id);
			}
		}
	}

	/* CONNECT: Any POST data is for the origin server only.  */
	if (!use_connect && uri->post) {
		/* We search for first '\n' in uri->post to get content type
		 * as set by get_form_uri(). This '\n' is dropped if any
		 * and replaced by correct '\r\n' termination here. */
		unsigned char *postend = strchr((const char *)uri->post, '\n');
		struct connection_state error;

		if (postend) {
			add_to_string(&header, "Content-Type: ");
			add_bytes_to_string(&header, uri->post, postend - uri->post);
			add_crlf_to_string(&header);
		}

		post_data = postend ? postend + 1 : uri->post;
		if (!open_http_post(&http->post, post_data, &error)) {
			http_end_request(conn, error, 0);
			done_string(&header);
			return;
		}
		add_format_to_string(&header, "Content-Length: "
				     "%" OFF_PRINT_FORMAT "\x0D\x0A",
				     (off_print_T)
				     http->post.total_upload_length);
	}

#ifdef CONFIG_COOKIES
	/* CONNECT: Cookies are for the origin server only.  */
	if (!use_connect) {
		struct string *cookies = send_cookies(uri);

		if (cookies) {
			add_to_string(&header, "Cookie: ");
			add_string_to_string(&header, cookies);
			add_crlf_to_string(&header);
			done_string(cookies);
		}
	}
#endif

	add_crlf_to_string(&header);

	/* CONNECT: Any POST data is for the origin server only.
	 * This was already checked above and post_data is NULL
	 * in that case.  Verified with an assertion below.  */
	if (post_data) {
		assert(!use_connect); /* see comment above */

		socket->state = SOCKET_END_ONCLOSE;
		if (!conn->http_upload_progress && http->post.file_count)
			conn->http_upload_progress = init_progress(0);
		write_to_socket(socket, header.source, header.length,
				connection_state(S_TRANS),
				send_more_post_data);
	} else
		request_from_socket(socket, header.source, header.length,
				    connection_state(S_SENT),
				    SOCKET_END_ONCLOSE, http_got_header);
	done_string(&header);
}

#undef POST_BUFFER_SIZE


static unsigned char *
decompress_data(struct connection *conn, unsigned char *data, int len,
		int *new_len)
{
	*new_len = 0; /* new_len must be zero if we would ever return NULL */

	if (!conn->stream) {
		conn->stream = open_encoded(-1, conn->content_encoding);
		if (!conn->stream) return NULL;
	}

	return decode_encoded_buffer(conn->stream, conn->content_encoding, data, len, new_len);
}

static int
is_line_in_buffer(struct read_buffer *rb)
{
	int l;

	for (l = 0; l < rb->length; l++) {
		unsigned char a0 = rb->data[l];

		if (a0 == ASCII_LF)
			return l + 1;
		if (a0 == ASCII_CR) {
			if (rb->data[l + 1] == ASCII_LF
			    && l < rb->length - 1)
				return l + 2;
			if (l == rb->length - 1)
				return 0;
		}
		if (a0 < ' ')
			return -1;
	}
	return 0;
}

static void read_http_data(struct socket *socket, struct read_buffer *rb);

static void
read_more_http_data(struct connection *conn, struct read_buffer *rb,
                    int already_got_anything)
{
	struct connection_state state = already_got_anything
		? connection_state(S_TRANS) : conn->state;

	read_from_socket(conn->socket, rb, state, read_http_data);
}

static void
read_http_data_done(struct connection *conn)
{
	struct http_connection_info *http = conn->info;

	/* There's no content but an error so just print
	 * that instead of nothing. */
	if (!conn->from) {
		if (http->code >= 400) {
			http_error_document(conn, http->code);

		} else {
			/* This is not an error, thus fine. No need generate any
			 * document, as this may be empty and it's not a problem.
			 * In case of 3xx, we're probably just getting kicked to
			 * another page anyway. And in case of 2xx, the document
			 * may indeed be empty and thus the user should see it so. */
		}
	}

	http_end_request(conn, connection_state(S_OK), 0);
}

/* Returns:
 * -1 on error
 * 0 if more to read
 * 1 if done
 */
static int
read_chunked_http_data(struct connection *conn, struct read_buffer *rb)
{
	struct http_connection_info *http = conn->info;
	int total_data_len = 0;

	while (1) {
		/* Chunked. Good luck! */
		/* See RFC2616, section 3.6.1. Basically, it looks like:
		 * 1234 ; a = b ; c = d\r\n
		 * aklkjadslkfjalkfjlkajkljfdkljdsfkljdf*1234\r\n
		 * 0\r\n
		 * \r\n */
		if (http->chunk_remaining == CHUNK_DATA_END) {
			int l = is_line_in_buffer(rb);

			if (l) {
				if (l == -1) {
					/* Invalid character in buffer. */
					return -1;
				}

				/* Remove everything to the EOLN. */
				kill_buffer_data(rb, l);
				if (l <= 2) {
					/* Empty line. */
					return 2;
				}
				continue;
			}

		} else if (http->chunk_remaining == CHUNK_SIZE) {
			int l = is_line_in_buffer(rb);

			if (l) {
				unsigned char *de;
				int n = 0;

				if (l != -1) {
					errno = 0;
					n = strtol(rb->data, (char **) &de, 16);
					if (errno || !*de) {
						return -1;
					}
				}

				if (l == -1 || de == rb->data) {
					return -1;
				}

				/* Remove everything to the EOLN. */
				kill_buffer_data(rb, l);
				http->chunk_remaining = n;
				if (!http->chunk_remaining)
					http->chunk_remaining = CHUNK_ZERO_SIZE;
				continue;
			}

		} else {
			int data_len;
			int zero = (http->chunk_remaining == CHUNK_ZERO_SIZE);
			int len = zero ? 0 : http->chunk_remaining;

			/* Maybe everything necessary didn't come yet.. */
			int_upper_bound(&len, rb->length);
			conn->received += len;

			if (http->chunk_remaining > 0) http->chunk_remaining -= len;
			if (conn->content_encoding == ENCODING_NONE) {
				data_len = len;
				if (add_fragment(conn->cached, conn->from, rb->data, len) == 1)
					conn->tries = 0;
			} else {
				unsigned char *data = decompress_data(conn, rb->data, len, &data_len);

				if (add_fragment(conn->cached, conn->from, data, data_len) == 1)
					conn->tries = 0;

				mem_free_if(data);
				if (zero || !http->length) shutdown_connection_stream(conn);
			}

			conn->from += data_len;
			total_data_len += data_len;

			kill_buffer_data(rb, len);

			if (zero) {
				/* Last chunk has zero length, so this is last
				 * chunk, we finished decompression just now
				 * and now we can happily finish reading this
				 * stuff. */
				http->chunk_remaining = CHUNK_DATA_END;
				continue;
			}

			if (!http->chunk_remaining && rb->length > 0) {
				/* Eat newline succeeding each chunk. */
				if (rb->data[0] == ASCII_LF) {
					kill_buffer_data(rb, 1);
				} else {
					if (rb->data[0] != ASCII_CR
					    || (rb->length >= 2
						&& rb->data[1] != ASCII_LF)) {
						return -1;
					}
					if (rb->length < 2) break;
					kill_buffer_data(rb, 2);
				}
				http->chunk_remaining = CHUNK_SIZE;
				continue;
			}
		}
		break;
	}

	/* More to read. */
	return !!total_data_len;
}

/* Returns 0 if more data, 1 if done. */
static int
read_normal_http_data(struct connection *conn, struct read_buffer *rb)
{
	struct http_connection_info *http = conn->info;
	int data_len;
	int len = rb->length;

	if (http->length >= 0 && http->length < len) {
		/* We won't read more than we have to go. */
		len = http->length;
	}

	conn->received += len;
	if (http->length > 0) http->length -= len;

	if (conn->content_encoding == ENCODING_NONE) {
		data_len = len;
		if (add_fragment(conn->cached, conn->from, rb->data, data_len) == 1)
			conn->tries = 0;
	} else {
		unsigned char *data;
finish:
		data = decompress_data(conn, rb->data, len, &data_len);

		if (!data && !http->length && len) {
			kill_buffer_data(rb, len);
			len = 0;
			goto finish;
		}

		if (add_fragment(conn->cached, conn->from, data, data_len) == 1)
			conn->tries = 0;

		mem_free_if(data);
		if (!http->length) shutdown_connection_stream(conn);
	}

	conn->from += data_len;

	kill_buffer_data(rb, len);

	if (!http->length && (conn->socket->state == SOCKET_RETRY_ONCLOSE
		|| conn->socket->state == SOCKET_CLOSED)) {
		return 2;
	}

	return !!data_len;
}

static void
read_http_data(struct socket *socket, struct read_buffer *rb)
{
	struct connection *conn = socket->conn;
	struct http_connection_info *http = conn->info;
	int ret;

	if (socket->state == SOCKET_CLOSED) {
		if (conn->content_encoding) {
			/* Flush decompression first. */
			http->length = 0;
		} else {
			read_http_data_done(conn);
			return;
		}
	}

	if (http->length != LEN_CHUNKED) {
		ret = read_normal_http_data(conn, rb);

	} else {
		ret = read_chunked_http_data(conn, rb);
	}

	switch (ret) {
	case 0:
		read_more_http_data(conn, rb, 0);
		break;
	case 1:
		read_more_http_data(conn, rb, 1);
		break;
	case 2:
		read_http_data_done(conn);
		break;
	default:
		assertm(ret == -1, "Unexpected return value: %d", ret);
		abort_connection(conn, connection_state(S_HTTP_ERROR));
	}
}

/* Returns offset of the header end, zero if more data is needed, -1 when
 * incorrect data was received, -2 if this is HTTP/0.9 and no header is to
 * come. */
static int
get_header(struct read_buffer *rb)
{
	int i;

	/* XXX: We will have to do some guess about whether an HTTP header is
	 * coming or not, in order to support HTTP/0.9 reply correctly. This
	 * means a little code duplication with get_http_code(). --pasky */
	if (rb->length > 4 && c_strncasecmp(rb->data, "HTTP/", 5))
		return -2;

	for (i = 0; i < rb->length; i++) {
		unsigned char a0 = rb->data[i];
		unsigned char a1 = rb->data[i + 1];

		if (a0 == 0) {
			rb->data[i] = ' ';
			continue;
		}
		if (a0 == ASCII_LF && a1 == ASCII_LF
		    && i < rb->length - 1)
			return i + 2;
		if (a0 == ASCII_CR && i < rb->length - 3) {
			if (a1 == ASCII_CR) continue;
			if (a1 != ASCII_LF) return -1;
			if (rb->data[i + 2] == ASCII_CR) {
				if (rb->data[i + 3] != ASCII_LF) return -1;
				return i + 4;
			}
		}
	}

	return 0;
}

/* returns 1 if we need retry the connection (for negotiate-auth only) */
static int
check_http_authentication(struct connection *conn, struct uri *uri,
		unsigned char *header, unsigned char *header_field)
{
	unsigned char *str, *d;
	int ret = 0;

	d = parse_header(header, header_field, &str);
	while (d) {
		if (!c_strncasecmp(d, "Basic", 5)) {
			unsigned char *realm = get_header_param(d, "realm");

			if (realm) {
				add_auth_entry(uri, realm, NULL, NULL, 0);
				mem_free(realm);
				mem_free(d);
				break;
			}
		} else if (!c_strncasecmp(d, "Digest", 6)) {
			unsigned char *realm = get_header_param(d, "realm");
			unsigned char *nonce = get_header_param(d, "nonce");
			unsigned char *opaque = get_header_param(d, "opaque");

			add_auth_entry(uri, realm, nonce, opaque, 1);

			mem_free_if(realm);
			mem_free_if(nonce);
			mem_free_if(opaque);
			mem_free(d);
			break;
		}
#ifdef CONFIG_GSSAPI
		else if (!c_strncasecmp(d, HTTPNEG_GSS_STR, HTTPNEG_GSS_STRLEN)) {
			if (http_negotiate_input(conn, uri, HTTPNEG_GSS, str)==0)
				ret = 1;
			mem_free(d);
			break;
		}
		else if (!c_strncasecmp(d, HTTPNEG_NEG_STR, HTTPNEG_NEG_STRLEN)) {
			if (http_negotiate_input(conn, uri, HTTPNEG_NEG, str)==0)
				ret = 1;
			mem_free(d);
			break;
		}
#endif
		mem_free(d);
		d = parse_header(str, header_field, &str);
	}
	return ret;
}


void
http_got_header(struct socket *socket, struct read_buffer *rb)
{
	struct connection *conn = socket->conn;
	struct http_connection_info *http = conn->info;
	unsigned char *head;
#ifdef CONFIG_COOKIES
	unsigned char *cookie, *ch;
#endif
	unsigned char *d;
	struct uri *uri = conn->proxied_uri; /* Set to the real uri */
	struct http_version version = { 0, 9 };
	struct connection_state state = (!is_in_state(conn->state, S_PROC)
					 ? connection_state(S_GETH)
					 : connection_state(S_PROC));
	int a, h = 200;
	int cf;

	if (socket->state == SOCKET_CLOSED) {
		if (!conn->tries && uri->host) {
			if (http->bl_flags & SERVER_BLACKLIST_NO_CHARSET) {
				del_blacklist_entry(uri, SERVER_BLACKLIST_NO_CHARSET);
			} else {
				add_blacklist_entry(uri, SERVER_BLACKLIST_NO_CHARSET);
				conn->tries = -1;
			}
		}
		retry_connection(conn, connection_state(S_CANT_READ));
		return;
	}
	socket->state = SOCKET_RETRY_ONCLOSE;

again:
	a = get_header(rb);
	if (a == -1) {
		abort_connection(conn, connection_state(S_HTTP_ERROR));
		return;
	}
	if (!a) {
		read_from_socket(conn->socket, rb, state, http_got_header);
		return;
	}
	/* a == -2 from get_header means HTTP/0.9.  In that case, skip
	 * the get_http_code call; @h and @version have already been
	 * initialized with the right values.  */
	if (a == -2) a = 0;
	if ((a && get_http_code(rb, &h, &version))
	    || h == 101) {
		abort_connection(conn, connection_state(S_HTTP_ERROR));
		return;
	}

	/* When no header, HTTP/0.9 document. That's always text/html,
	 * according to
	 * http://www.w3.org/Protocols/HTTP/AsImplemented.html. */
	/* FIXME: This usage of fake protocol headers for setting up the
	 * content type has been obsoleted by the @content_type member of
	 * {struct cache_entry}. */
	head = (a ? memacpy(rb->data, a)
		  : stracpy("\r\nContent-Type: text/html\r\n"));
	if (!head) {
		abort_connection(conn, connection_state(S_OUT_OF_MEM));
		return;
	}

	if (check_http_server_bugs(uri, http, head)) {
		mem_free(head);
		retry_connection(conn, connection_state(S_RESTART));
		return;
	}

#ifdef CONFIG_CGI
	if (uri->protocol == PROTOCOL_FILE) {
		/* ``Status'' is not a standard HTTP header field although some
		 * HTTP servers like www.php.net uses it for some reason. It should
		 * only be used for CGI scripts so that it does not interfere
		 * with status code depended handling for ``normal'' HTTP like
		 * redirects. */
		d = parse_header(head, "Status", NULL);
		if (d) {
			int h2 = atoi(d);

			mem_free(d);
			if (h2 >= 100 && h2 < 600) h = h2;
			if (h == 101) {
				mem_free(head);
				abort_connection(conn, connection_state(S_HTTP_ERROR));
				return;
			}
		}
	}
#endif

#ifdef CONFIG_COOKIES
	ch = head;
	while ((cookie = parse_header(ch, "Set-Cookie", &ch))) {
		set_cookie(uri, cookie);
		mem_free(cookie);
	}
#endif
	http->code = h;

	if (h == 100) {
		mem_free(head);
		state = connection_state(S_PROC);
		kill_buffer_data(rb, a);
		goto again;
	}
	if (h < 200) {
		mem_free(head);
		abort_connection(conn, connection_state(S_HTTP_ERROR));
		return;
	}
	if (h == 304) {
		mem_free(head);
		http_end_request(conn, connection_state(S_OK), 1);
		return;
	}
	if (h == 204) {
		mem_free(head);
		http_end_request(conn, connection_state(S_HTTP_204), 0);
		return;
	}
	if (h == 200 && connection_is_https_proxy(conn) && !conn->socket->ssl) {
#ifdef CONFIG_SSL
		mem_free(head);
		socket->need_ssl = 1;
		complete_connect_socket(socket, uri, http_send_header);
#else
		abort_connection(conn, connection_state(S_SSL_ERROR));
#endif
		return;
	}

	conn->cached = get_cache_entry(conn->uri);
	if (!conn->cached) {
		mem_free(head);
		abort_connection(conn, connection_state(S_OUT_OF_MEM));
		return;
	}
	conn->cached->cgi = conn->cgi;
	mem_free_set(&conn->cached->head, head);

	if (!get_opt_bool("document.cache.ignore_cache_control", NULL)) {
		struct cache_entry *cached = conn->cached;

		/* I am not entirely sure in what order we should process these
		 * headers and if we should still process Cache-Control max-age
		 * if we already set max age to date mentioned in Expires.
		 * --jonas */
		/* Ensure that when ever cached->max_age is set, cached->expired
		 * is also set, so the cache management knows max_age contains a
		 * valid time. If on the other hand no caching is requested
		 * cached->expire should be set to zero.  */
		if ((d = parse_header(cached->head, "Expires", NULL))) {
			/* Convert date to seconds. */
			time_t expires = parse_date(&d, NULL, 0, 1);

			mem_free(d);

			if (expires && cached->cache_mode != CACHE_MODE_NEVER) {
				timeval_from_seconds(&cached->max_age, expires);
				cached->expire = 1;
			}
		}

		if ((d = parse_header(cached->head, "Pragma", NULL))) {
			if (strstr((const char *)d, "no-cache")) {
				cached->cache_mode = CACHE_MODE_NEVER;
				cached->expire = 0;
			}
			mem_free(d);
		}

		if (cached->cache_mode != CACHE_MODE_NEVER
		    && (d = parse_header(cached->head, "Cache-Control", NULL))) {
			if (strstr((const char *)d, "no-cache") || strstr((const char *)d, "must-revalidate")) {
				cached->cache_mode = CACHE_MODE_NEVER;
				cached->expire = 0;

			} else  {
				unsigned char *pos = strstr((const char *)d, "max-age=");

				assert(cached->cache_mode != CACHE_MODE_NEVER);

				if (pos) {
					/* Grab the number of seconds. */
					timeval_T max_age;

					timeval_from_seconds(&max_age, atol(pos + 8));
					timeval_now(&cached->max_age);
					timeval_add_interval(&cached->max_age, &max_age);

					cached->expire = 1;
				}
			}

			mem_free(d);
		}
	}

	/* XXX: Is there some reason why NOT to follow the Location header
	 * for any status? If the server didn't mean it, it wouldn't send
	 * it, after all...? --pasky */
	if (h == 201 || h == 301 || h == 302 || h == 303 || h == 307) {
		d = parse_header(conn->cached->head, "Location", NULL);
		if (d) {
			int use_get_method = (h == 303);

			/* A note from RFC 2616 section 10.3.3:
			 * RFC 1945 and RFC 2068 specify that the client is not
			 * allowed to change the method on the redirected
			 * request. However, most existing user agent
			 * implementations treat 302 as if it were a 303
			 * response, performing a GET on the Location
			 * field-value regardless of the original request
			 * method. */
			/* So POST must not be redirected to GET, but some
			 * BUGGY message boards rely on it :-( */
	    		if (h == 302
			    && get_opt_bool("protocol.http.bugs.broken_302_redirect", NULL))
				use_get_method = 1;

			redirect_cache(conn->cached, d, use_get_method, -1);
			mem_free(d);
		}
	}

	if (h == 401) {
		if (check_http_authentication(conn, uri,
				conn->cached->head, "WWW-Authenticate")) {
			retry_connection(conn, connection_state(S_RESTART));
			return;
		}

	}
	if (h == 407) {
		unsigned char *str;
		int restart = 0;

		d = parse_header(conn->cached->head, "Proxy-Authenticate", &str);
		while (d) {
			if (!c_strncasecmp(d, "Basic", 5)) {
				unsigned char *realm = get_header_param(d, "realm");

				if (realm) {
					mem_free_set(&proxy_auth.realm, realm);
					proxy_auth.digest = 0;
					mem_free(d);
					break;
				}

			} else if (!c_strncasecmp(d, "Digest", 6)) {
				unsigned char *realm = get_header_param(d, "realm");
				unsigned char *nonce = get_header_param(d, "nonce");
				unsigned char *opaque = get_header_param(d, "opaque");
				unsigned char *stale = get_header_param(d, "stale");

				if (stale) {
					if (strcasecmp(stale, "true")) restart = 1;
					else restart = 0;
					mem_free(stale);
				}
				mem_free_set(&proxy_auth.realm, realm);
				mem_free_set(&proxy_auth.nonce, nonce);
				mem_free_set(&proxy_auth.opaque, opaque);
				if (proxy_auth.digest == 0) restart = 1;
				proxy_auth.digest = 1;

				mem_free(d);
				break;
			}

			mem_free(d);
			d = parse_header(str, "Proxy-Authenticate", &str);
		}
		if (restart) {
			retry_connection(conn, connection_state(S_RESTART));
			return;	
		}
	}

	kill_buffer_data(rb, a);
	http->close = 0;
	http->length = -1;
	http->recv_version = version;

	if ((d = parse_header(conn->cached->head, "Connection", NULL))
	     || (d = parse_header(conn->cached->head, "Proxy-Connection", NULL))) {
		if (!c_strcasecmp(d, "close")) http->close = 1;
		mem_free(d);
	} else if (PRE_HTTP_1_1(version)) {
		http->close = 1;
	}

	cf = conn->from;
	conn->from = 0;
	d = parse_header(conn->cached->head, "Content-Range", NULL);
	if (d) {
		if (strlen(d) > 6) {
			d[5] = 0;
			if (isdigit(d[6]) && !c_strcasecmp(d, "bytes")) {
				int f;

				errno = 0;
				f = strtol(d + 6, NULL, 10);

				if (!errno && f >= 0) conn->from = f;
			}
		}
		mem_free(d);
	}
	if (cf && !conn->from && !conn->unrestartable) conn->unrestartable = 1;
	if ((conn->progress->start <= 0 && conn->from > cf) || conn->from < 0) {
		/* We don't want this if conn->progress.start because then conn->from will
		 * be probably value of conn->progress.start, while cf is 0. */
		abort_connection(conn, connection_state(S_HTTP_ERROR));
		return;
	}

#if 0
	{
		struct status *s;
		foreach (s, conn->downloads) {
			fprintf(stderr, "conn %p status %p pri %d st %d er %d :: ce %s",
				conn, s, s->pri, s->state, s->prev_error,
				s->cached ? s->cached->url : (unsigned char *) "N-U-L-L");
		}
	}
#endif

	if (conn->progress->start >= 0) {
		/* Update to the real value which we've got from Content-Range. */
		conn->progress->seek = conn->from;
	}
	conn->progress->start = conn->from;

	d = parse_header(conn->cached->head, "Content-Length", NULL);
	if (d) {
		unsigned char *ep;
		long long l;

		errno = 0;
		l = strtoll(d, (char **) &ep, 10);

		if (!errno && !*ep && l >= 0) {
			if (!http->close || POST_HTTP_1_0(version))
				http->length = l;
			conn->est_length = conn->from + l;
		}
		mem_free(d);
	}

	if (!conn->unrestartable) {
		d = parse_header(conn->cached->head, "Accept-Ranges", NULL);

		if (d) {
			if (!c_strcasecmp(d, "none"))
				conn->unrestartable = 1;
			mem_free(d);
		} else {
			if (!conn->from)
				conn->unrestartable = 1;
		}
	}

	d = parse_header(conn->cached->head, "Transfer-Encoding", NULL);
	if (d) {
		if (!c_strcasecmp(d, "chunked")) {
			http->length = LEN_CHUNKED;
			http->chunk_remaining = CHUNK_SIZE;
		}
		mem_free(d);
	}
	if (!http->close && http->length == -1) http->close = 1;

	d = parse_header(conn->cached->head, "Last-Modified", NULL);
	if (d) {
		if (conn->cached->last_modified && c_strcasecmp(conn->cached->last_modified, d)) {
			delete_entry_content(conn->cached);
			if (conn->from) {
				conn->from = 0;
				mem_free(d);
				retry_connection(conn, connection_state(S_MODIFIED));
				return;
			}
		}
		if (!conn->cached->last_modified) conn->cached->last_modified = d;
		else mem_free(d);
	}
	if (!conn->cached->last_modified) {
		d = parse_header(conn->cached->head, "Date", NULL);
		if (d) conn->cached->last_modified = d;
	}

	/* FIXME: Parse only if HTTP/1.1 or later? --Zas */
	d = parse_header(conn->cached->head, "ETag", NULL);
	if (d) {
		if (conn->cached->etag) {
			unsigned char *old_tag = conn->cached->etag;
			unsigned char *new_tag = d;

			/* http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.19 */

			if (new_tag[0] == 'W' && new_tag[1] == '/')
				new_tag += 2;

			if (old_tag[0] == 'W' && old_tag[1] == '/')
				old_tag += 2;

			if (strcmp(new_tag, old_tag)) {
				delete_entry_content(conn->cached);
				if (conn->from) {
					conn->from = 0;
					mem_free(d);
					retry_connection(conn, connection_state(S_MODIFIED));
					return;
				}
			}
		}

		if (!conn->cached->etag)
			conn->cached->etag = d;
		else
			mem_free(d);
	}

	d = parse_header(conn->cached->head, "Content-Encoding", NULL);
	if (d) {
#if defined(CONFIG_GZIP) || defined(CONFIG_BZIP2) || defined(CONFIG_LZMA) || defined(CONFIG_BROTLI) || defined(CONFIG_ZSTD)
		unsigned char *extension = get_extension_from_uri(uri);
		enum stream_encoding file_encoding;

		file_encoding = extension ? guess_encoding(extension) : ENCODING_NONE;
		mem_free_if(extension);
#endif
		/* If the content is encoded, we want to preserve the encoding
		 * if it is implied by the extension, so that saving the URI
		 * will leave the saved file with the correct encoding. */
#ifdef CONFIG_GZIP
		if (file_encoding != ENCODING_GZIP
		    && (!c_strcasecmp(d, "gzip") || !c_strcasecmp(d, "x-gzip")))
		    	conn->content_encoding = ENCODING_GZIP;
		if (!c_strcasecmp(d, "deflate") || !c_strcasecmp(d, "x-deflate"))
			conn->content_encoding = ENCODING_DEFLATE;
#endif

#ifdef CONFIG_ZSTD
		if (file_encoding != ENCODING_ZSTD
		    && (!c_strcasecmp(d, "zstd")))
			conn->content_encoding = ENCODING_ZSTD;
#endif

#ifdef CONFIG_BROTLI
		if (file_encoding != ENCODING_BROTLI
		    && (!c_strcasecmp(d, "br")))
			conn->content_encoding = ENCODING_BROTLI;
#endif

#ifdef CONFIG_BZIP2
		if (file_encoding != ENCODING_BZIP2
		    && (!c_strcasecmp(d, "bzip2") || !c_strcasecmp(d, "x-bzip2")))
			conn->content_encoding = ENCODING_BZIP2;
#endif

#ifdef CONFIG_LZMA
		if (file_encoding != ENCODING_LZMA
		    && (!c_strcasecmp(d, "lzma") || !c_strcasecmp(d, "x-lzma")))
			conn->content_encoding = ENCODING_LZMA;
#endif
		mem_free(d);
	}

	if (conn->content_encoding != ENCODING_NONE) {
		mem_free_if(conn->cached->encoding_info);
		conn->cached->encoding_info = stracpy(get_encoding_name(conn->content_encoding));
	}

	if (http->length == -1 || http->close)
		socket->state = SOCKET_END_ONCLOSE;

	read_http_data(socket, rb);
}
