luasec: added openssl 1.1.0 compatibility

This patch was a backport of 0.7alpha, without changing version 0.6
functionality.

Signed-off-by: Eneas U de Queiroz <cote2004-github@yahoo.com>

lang/luasec/Makefile

@@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=luasec
 PKG_VERSION:=0.6
-PKG_RELEASE:=1
+PKG_RELEASE:=2
 
 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz
 PKG_SOURCE_URL:=https://github.com/brunoos/luasec/archive/

lang/luasec/patches/010-openssl-1.1-compatibility.patch

@@ -0,0 +1,180 @@
+--- a/src/context.c
++++ b/src/context.c
+@@ -24,7 +24,7 @@
+ #include "context.h"
+ #include "options.h"
+ 
+-#ifndef OPENSSL_NO_ECDH
++#ifndef OPENSSL_NO_EC
+ #include <openssl/ec.h>
+ #include "ec.h"
+ #endif
+@@ -35,10 +35,6 @@ typedef const SSL_METHOD LSEC_SSL_METHOD
+ typedef       SSL_METHOD LSEC_SSL_METHOD;
+ #endif
+ 
+-#if OPENSSL_VERSION_NUMBER>=0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
+-#define SSLv23_method() TLS_method()
+-#endif
+-
+ /*-- Compat - Lua 5.1 --------------------------------------------------------*/
+ 
+ #if (LUA_VERSION_NUM == 501)
+@@ -304,7 +300,7 @@ static int verify_cb(int preverify_ok, X
+   return (verify & LSEC_VERIFY_CONTINUE ? 1 : preverify_ok);
+ }
+ 
+-#ifndef OPENSSL_NO_ECDH
++#ifndef OPENSSL_NO_EC
+ static EC_KEY *find_ec_key(const char *str)
+ {
+   p_ec ptr;
+@@ -565,7 +561,7 @@ static int set_dhparam(lua_State *L)
+ /**
+  * Set elliptic curve.
+  */
+-#ifdef OPENSSL_NO_ECDH
++#ifdef OPENSSL_NO_EC
+ static int set_curve(lua_State *L)
+ {
+   lua_pushboolean(L, 0);
+--- a/src/ssl.c
++++ b/src/ssl.c
+@@ -31,6 +31,13 @@
+ #include "context.h"
+ #include "ssl.h"
+ 
++
++#if defined(LIBRESSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER<0x10100000L
++#define SSL_is_server(s) (s->server)
++#define X509_up_ref(c)   CRYPTO_add(&c->references, 1, CRYPTO_LOCK_X509)
++#endif
++
++
+ /**
+  * Underline socket error.
+  */
+@@ -406,7 +413,9 @@ static int meth_want(lua_State *L)
+  */
+ static int meth_compression(lua_State *L)
+ {
+-#if !defined(OPENSSL_NO_COMP)
++#ifdef OPENSSL_NO_COMP
++  const void *comp;
++#else
+   const COMP_METHOD *comp;
+ #endif
+   p_ssl ssl = (p_ssl)luaL_checkudata(L, 1, "SSL:Connection");
+@@ -415,15 +424,11 @@ static int meth_compression(lua_State *L
+     lua_pushstring(L, "closed");
+     return 2;
+   }
+-#if !defined(OPENSSL_NO_COMP)
+   comp = SSL_get_current_compression(ssl->ssl);
+   if (comp)
+     lua_pushstring(L, SSL_COMP_get_name(comp));
+   else
+     lua_pushnil(L);
+-#else
+-  lua_pushnil(L);
+-#endif
+   return 1;
+ }
+ 
+@@ -461,7 +466,7 @@ static int meth_getpeercertificate(lua_S
+   /* In a server-context, the stack doesn't contain the peer cert,
+    * so adjust accordingly.
+    */
+-  if (ssl->ssl->server)
++  if (SSL_is_server(ssl->ssl))
+     --n;
+   certs = SSL_get_peer_cert_chain(ssl->ssl);
+   if (n >= sk_X509_num(certs)) {
+@@ -471,7 +476,7 @@ static int meth_getpeercertificate(lua_S
+   cert = sk_X509_value(certs, n);
+   /* Increment the reference counting of the object. */
+   /* See SSL_get_peer_certificate() source code.     */
+-  CRYPTO_add(&cert->references, 1, CRYPTO_LOCK_X509);
++  X509_up_ref(cert);
+   lsec_pushx509(L, cert);
+   return 1;
+ }
+@@ -493,7 +498,7 @@ static int meth_getpeerchain(lua_State *
+     return 2;
+   }
+   lua_newtable(L);
+-  if (ssl->ssl->server) {
++  if (SSL_is_server(ssl->ssl)) {
+     lsec_pushx509(L, SSL_get_peer_certificate(ssl->ssl));
+     lua_rawseti(L, -2, idx++);
+   }
+@@ -503,7 +508,7 @@ static int meth_getpeerchain(lua_State *
+     cert = sk_X509_value(certs, i);
+     /* Increment the reference counting of the object. */
+     /* See SSL_get_peer_certificate() source code.     */
+-    CRYPTO_add(&cert->references, 1, CRYPTO_LOCK_X509);
++    X509_up_ref(cert);
+     lsec_pushx509(L, cert);
+     lua_rawseti(L, -2, idx++);
+   }
+--- a/src/x509.c
++++ b/src/x509.c
+@@ -32,6 +32,17 @@
+ 
+ #include "x509.h"
+ 
++
++/*
++ * ASN1_STRING_data is deprecated in OpenSSL 1.1.0
++ */
++#if OPENSSL_VERSION_NUMBER>=0x1010000fL && !defined(LIBRESSL_VERSION_NUMBER)
++#define LSEC_ASN1_STRING_data(x) ASN1_STRING_get0_data(x)
++#else
++#define LSEC_ASN1_STRING_data(x) ASN1_STRING_data(x)
++#endif
++
++
+ static const char* hex_tab = "0123456789abcdef";
+ 
+ /**
+@@ -146,7 +157,7 @@ static void push_asn1_string(lua_State*
+   }
+   switch (encode) {
+   case LSEC_AI5_STRING:
+-    lua_pushlstring(L, (char*)ASN1_STRING_data(string),
++    lua_pushlstring(L, (char*)LSEC_ASN1_STRING_data(string),
+                        ASN1_STRING_length(string));
+     break;
+   case LSEC_UTF8_STRING:
+@@ -182,7 +193,7 @@ static void push_asn1_ip(lua_State *L, A
+ {
+   int af;
+   char dst[INET6_ADDRSTRLEN];
+-  unsigned char *ip = ASN1_STRING_data(string);
++  unsigned char *ip = (unsigned char*)LSEC_ASN1_STRING_data(string);
+   switch(ASN1_STRING_length(string)) {
+   case 4:
+     af = AF_INET;
+@@ -293,11 +304,11 @@ int meth_extensions(lua_State* L)
+       break;
+ 
+     /* Push ret[oid] */
+-    push_asn1_objname(L, extension->object, 1);
++    push_asn1_objname(L, X509_EXTENSION_get_object(extension), 1);
+     push_subtable(L, -2);
+ 
+     /* Set ret[oid].name = name */
+-    push_asn1_objname(L, extension->object, 0);
++    push_asn1_objname(L, X509_EXTENSION_get_object(extension), 0);
+     lua_setfield(L, -2, "name");
+ 
+     n_general_names = sk_GENERAL_NAME_num(values);
+@@ -404,7 +415,7 @@ static int meth_pubkey(lua_State* L)
+     bytes = BIO_get_mem_data(bio, &data);
+     if (bytes > 0) {
+       lua_pushlstring(L, data, bytes);
+-      switch(EVP_PKEY_type(pkey->type)) {
++      switch(EVP_PKEY_base_id(pkey)) {
+         case EVP_PKEY_RSA:
+           lua_pushstring(L, "RSA");
+           break;