[mod_openssl] fix warnings and compile breaks with openssl 1.1.0

Change-Id: Ia69e8192004208a9e55246196b5b64d39cd53a66

src/modules/mod_openssl.c

@@ -240,6 +240,7 @@ static void openssl_setenv_X509_add_entries(liVRequest *vr, X509 *x509, const gc
 
 	X509_NAME *xn = X509_get_subject_name(x509);
 	X509_NAME_ENTRY *xe;
+	ASN1_STRING *xes;
 	const char * xobjsn;
 
 	g_string_truncate(k, 0);
@@ -251,7 +252,9 @@ static void openssl_setenv_X509_add_entries(liVRequest *vr, X509 *x509, const gc
 			continue;
 		g_string_truncate(k, prefix_len);
 		g_string_append(k, xobjsn);
-		li_environment_set(&vr->env, GSTR_LEN(k), (const gchar *)xe->value->data, xe->value->length);
+
+		xes = X509_NAME_ENTRY_get_data(xe);
+		li_environment_set(&vr->env, GSTR_LEN(k), (const gchar *)xes->data, xes->length);
 	}
 }
 
@@ -878,6 +881,7 @@ static void plugin_init(liServer *srv, liPlugin *p, gpointer userdata) {
 	p->setups = setups;
 }
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
 static GMutex** ssl_locks;
 
 static void ssl_lock_cb(int mode, int n, const char *file, int line) {
@@ -918,6 +922,15 @@ static void sslthread_free(void) {
 	g_slice_free1(sizeof(GMutex*) * n, ssl_locks);
 }
 
+#else
+
+static void sslthread_init(void) {
+}
+static void sslthread_free(void) {
+}
+
+#endif
+
 gboolean mod_openssl_init(liModules *mods, liModule *mod) {
 	MODULE_VERSION_CHECK(mods);
 
@@ -999,17 +1012,29 @@ static DH* load_dh_params_4096(void) {
 		0x05,
 		};
 
-	DH *dh = DH_new();
+	BIGNUM *dh_p, *dh_g;
+	DH *dh;
+
+	dh = DH_new();
 	if (NULL == dh) return NULL;
 
-	dh->p = BN_bin2bn(dh4096_p, sizeof(dh4096_p), NULL);
-	dh->g = BN_bin2bn(dh4096_g, sizeof(dh4096_g), NULL);
+	dh_p = BN_bin2bn(dh4096_p, sizeof(dh4096_p), NULL);
+	dh_g = BN_bin2bn(dh4096_g, sizeof(dh4096_g), NULL);
 
-	if (NULL == dh->p || NULL == dh->g) {
+	if (NULL == dh_p || NULL == dh_g) {
+		BN_free(dh_p);
+		BN_free(dh_g);
 		DH_free(dh);
 		return NULL;
 	}
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+	dh->p = dh_p;
+	dh->g = dh_g;
+#else
+	DH_set0_pqg(dh, dh_p, NULL, dh_g);
+#endif
+
 	return dh;
 }
 #endif

src/modules/openssl_filter.c

@@ -32,26 +32,7 @@ struct liOpenSSLFilter {
 
 #define BIO_TYPE_LI_STREAM (127|BIO_TYPE_SOURCE_SINK)
 
-static int stream_bio_write(BIO *bio, const char *buf, int len);
-static int stream_bio_read(BIO *bio, char *buf, int len);
-static int stream_bio_puts(BIO *bio, const char *str);
-static int stream_bio_gets(BIO *bio, char *str, int len);
-static long stream_bio_ctrl(BIO *bio, int cmd, long num, void *ptr);
-static int stream_bio_create(BIO *bio);
-static int stream_bio_destroy(BIO *bio);
-
-static BIO_METHOD chunkqueue_bio_method = {
-	BIO_TYPE_LI_STREAM,
-	"lighttpd stream glue",
-	stream_bio_write,
-	stream_bio_read,
-	stream_bio_puts,
-	stream_bio_gets,
-	stream_bio_ctrl,
-	stream_bio_create,
-	stream_bio_destroy,
-	NULL
-};
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
 
 static int stream_bio_write(BIO *bio, const char *buf, int len) {
 	liOpenSSLFilter *f = bio->ptr;
@@ -135,6 +116,136 @@ static int stream_bio_destroy(BIO *bio) {
 	return 1;
 }
 
+static BIO_METHOD chunkqueue_bio_method = {
+	BIO_TYPE_LI_STREAM,
+	"lighttpd stream glue",
+	stream_bio_write,
+	stream_bio_read,
+	stream_bio_puts,
+	stream_bio_gets,
+	stream_bio_ctrl,
+	stream_bio_create,
+	stream_bio_destroy,
+	NULL
+};
+
+static BIO* new_cq_bio(liOpenSSLFilter *f) {
+	BIO *bio = BIO_new(&chunkqueue_bio_method);
+	bio->ptr = f;
+}
+
+#else
+
+static int stream_bio_write(BIO *bio, const char *buf, int len) {
+	liOpenSSLFilter *f = BIO_get_data(bio);
+	liChunkQueue *cq;
+
+	errno = ECONNRESET;
+
+	if (NULL == f || NULL == f->crypt_source.out) return -1;
+	cq = f->crypt_source.out;
+	if (cq->is_closed) return -1;
+
+	li_chunkqueue_append_mem(cq, buf, len);
+	li_stream_notify_later(&f->crypt_source);
+
+	errno = 0;
+	return len;
+}
+static int stream_bio_read(BIO *bio, char *buf, int len) {
+	liOpenSSLFilter *f = BIO_get_data(bio);
+	liChunkQueue *cq;
+
+	errno = ECONNRESET;
+	BIO_clear_retry_flags(bio);
+
+	if (NULL == f || NULL == f->crypt_drain.out) return -1;
+
+	cq = f->crypt_drain.out;
+
+	if (0 == cq->length) {
+		if (cq->is_closed) {
+			errno = 0;
+			return 0;
+		} else {
+			errno = EAGAIN;
+			BIO_set_retry_read(bio);
+			return -1;
+		}
+	}
+
+	if (len > cq->length) len = cq->length;
+	if (!li_chunkqueue_extract_to_memory(cq, len, buf, NULL)) return -1;
+	li_chunkqueue_skip(cq, len);
+
+	errno = 0;
+	return len;
+}
+static int stream_bio_puts(BIO *bio, const char *str) {
+	return stream_bio_write(bio, str, strlen(str));
+}
+static int stream_bio_gets(BIO *bio, char *str, int len) {
+	UNUSED(bio); UNUSED(str); UNUSED(len);
+	return -1;
+}
+static long stream_bio_ctrl(BIO *bio, int cmd, long num, void *ptr) {
+	liOpenSSLFilter *f = BIO_get_data(bio);
+	UNUSED(num); UNUSED(ptr);
+
+	switch (cmd) {
+	case BIO_CTRL_FLUSH:
+		return 1;
+	case BIO_CTRL_PENDING:
+		if (NULL == f || NULL == f->crypt_drain.out) return 0;
+		return f->crypt_drain.out->length;
+	default:
+		return 0;
+	}
+}
+static int stream_bio_destroy(BIO *bio) {
+	liOpenSSLFilter *f = BIO_get_data(bio);
+	BIO_set_data(bio, NULL);
+	if (NULL != f) f->bio = NULL;
+	BIO_set_init(bio, 0);
+	return 1;
+}
+
+static BIO_METHOD* get_cq_bio_method(void) {
+	static gint once = 0;
+	static BIO_METHOD *m = NULL;
+
+	/* initialize once */
+	if (g_atomic_int_compare_and_exchange(&once, 0, 1)) {
+		m = BIO_meth_new(BIO_TYPE_LI_STREAM, "lighttpd stream glue");
+		LI_FORCE_ASSERT(NULL != m);
+
+		BIO_meth_set_write(m, stream_bio_write);
+		BIO_meth_set_read(m, stream_bio_read);
+		BIO_meth_set_puts(m, stream_bio_puts);
+		BIO_meth_set_gets(m, stream_bio_gets);
+		BIO_meth_set_ctrl(m, stream_bio_ctrl);
+		/* BIO_meth_set_create(m, stream_bio_create); */
+		BIO_meth_set_destroy(m, stream_bio_destroy);
+
+		g_atomic_int_set(&once, 2);
+	} else {
+		while (g_atomic_int_get(&once) != 2) { }
+	}
+
+	return m;
+}
+
+static BIO* new_cq_bio(liOpenSSLFilter *f) {
+	BIO_METHOD *m = get_cq_bio_method();
+	if (NULL == m) return NULL;
+	BIO *bio = BIO_new(m);
+	BIO_set_data(bio, f);
+	BIO_set_init(bio, 1);
+	return bio;
+}
+
+#endif
+
 static void f_close_ssl(liOpenSSLFilter *f) {
 	if (NULL != f->ssl && !f->closing) {
 		SSL *ssl;
@@ -273,7 +384,11 @@ static gboolean do_ssl_handshake(liOpenSSLFilter *f, gboolean writing) {
 	int r = SSL_do_handshake(f->ssl);
 	if (1 == r) {
 		f->initial_handshaked_finished = 1;
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
 		f->ssl->s3->flags |= SSL3_FLAGS_NO_RENEGOTIATE_CIPHERS;
+#else
+		/* hopefully openssl_info_callback catches this... */
+#endif
 		li_stream_acquire(&f->plain_source);
 		li_stream_acquire(&f->plain_drain);
 		f->callbacks->handshake_cb(f, f->callback_data, &f->plain_source, &f->plain_drain);
@@ -648,8 +763,7 @@ liOpenSSLFilter* li_openssl_filter_new(
 	f->ssl = ssl;
 	SSL_set_app_data(f->ssl, f);
 	SSL_set_info_callback(f->ssl, openssl_info_callback);
-	f->bio = BIO_new(&chunkqueue_bio_method);
-	f->bio->ptr = f;
+	f->bio = new_cq_bio(f);
 	SSL_set_bio(f->ssl, f->bio, f->bio);
 	/* BIO_set_callback(f->bio, BIO_debug_callback); */