I have an issue with openssl while connecting to TLS server with GOST server certificate.
/usr/local/bin/openssl s_client -CAfile /usr/share/ca-certificates/extra/VipNet-CA.crt -state -connect 10.0.99.50:443
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00007f1e7f3a1158 in BN_is_zero (a=0x0) at crypto/bn/bn_lib.c:922
922 return a->top == 0;
(gdb) bt full
#0 0x00007f1e7f3a1158 in BN_is_zero (a=0x0) at crypto/bn/bn_lib.c:922
No locals.
#1 0x00007f1e7e8a660c in gost_ec_verify (dgst=0x7ffc06897f50 "\035A\232\021\340\031~\334\372\206\360\026\334i}Sϧ\270\370\267^\037\020\272YQМ\260bZ\240\177\211\006\374\177", dgst_len=32, sig=0x2256210,
ec=0x2242a40) at /usr/src/engine/gost_ec_sign.c:348
ctx = 0x2256d80
group = 0x22432a0
order = 0x22590c0
md = 0x0
e = 0x22590d8
R = 0x2259150
v = 0x2259168
z1 = 0x22590f0
z2 = 0x2259108
sig_s = 0x0
sig_r = 0x0
X = 0x2259138
tmp = 0x2259120
C = 0x0
pub_key = 0x2241210
ok = 0
#2 0x00007f1e7e8abc64 in pkey_gost_ec_cp_verify (ctx=0x2256360,
sig=0x2254b70 "\247ԍ\246i-\340U(\241\351b\336\004<\r\221\244\205e$\365\337F\325\304\305\345\311\334\005\022\322\b\227\177\323\367\005\265\365\327\345e2\274\250\345~\342\264\301q\b·\353Hṙ\035\002\321P",
siglen=64, tbs=0x7ffc06897f50 "\035A\232\021\340\031~\334\372\206\360\026\334i}Sϧ\270\370\267^\037\020\272YQМ\260bZ\240\177\211\006\374\177", tbs_len=32) at /usr/src/engine/gost_pmeth.c:453
ok = 0
pub_key = 0x22424c0
s = 0x2256210
#3 0x00007f1e7f45c370 in EVP_PKEY_verify (ctx=0x2256360,
sig=0x2254b70 "\247ԍ\246i-\340U(\241\351b\336\004<\r\221\244\205e$\365\337F\325\304\305\345\311\334\005\022\322\b\227\177\323\367\005\265\365\327\345e2\274\250\345~\342\264\301q\b·\353Hṙ\035\002\321P",
siglen=64, tbs=0x7ffc06897f50 "\035A\232\021\340\031~\334\372\206\360\026\334i}Sϧ\270\370\267^\037\020\272YQМ\260bZ\240\177\211\006\374\177", tbslen=32) at crypto/evp/pmeth_fn.c:97
No locals.
#4 0x00007f1e7f45932c in EVP_DigestVerifyFinal (ctx=0x22561d0,
sig=0x2254b70 "\247ԍ\246i-\340U(\241\351b\336\004<\r\221\244\205e$\365\337F\325\304\305\345\311\334\005\022\322\b\227\177\323\367\005\265\365\327\345e2\274\250\345~\342\264\301q\b·\353Hṙ\035\002\321P",
siglen=64) at crypto/evp/m_sigver.c:168
md = "\035A\232\021\340\031~\334\372\206\360\026\334i}Sϧ\270\370\267^\037\020\272YQМ\260bZ\240\177\211\006\374\177\000\000\223\214F\177\036\177\000\000\000\000\000\000\244\000\000\000\340\332P\177\036\177\000"
r = 1
mdlen = 32
vctx = 0
#5 0x00007f1e7f360962 in ASN1_item_verify (it=0x7f1e7f797380 <X509_CINF_it>, a=0x223fb78, signature=0x223fb88, asn=0x223faf0, pkey=0x22424c0) at crypto/asn1/a_verify.c:172
ctx = 0x22561d0
buf_in = 0x22590c0 "\320l%\002"
ret = -1
inl = 1696
mdnid = 809
pknid = 811
#6 0x00007f1e7f4eeaac in X509_verify (a=0x223faf0, r=0x22424c0) at crypto/x509/x_all.c:26
No locals.
#7 0x00007f1e7f4e926a in internal_verify (ctx=0x2254ea0) at crypto/x509/x509_vfy.c:1719
pkey = 0x22424c0
n = 0
xi = 0x223e320
xs = 0x223faf0
#8 0x00007f1e7f4e61d6 in verify_chain (ctx=0x2254ea0) at crypto/x509/x509_vfy.c:233
err = 0
ok = 1
#9 0x00007f1e7f4e6417 in X509_verify_cert (ctx=0x2254ea0) at crypto/x509/x509_vfy.c:293
dane = 0x0
ret = 0
---Type <return> to continue, or q <return> to quit---
#10 0x00007f1e7f7dcc50 in ssl_verify_cert_chain (s=0x223e990, sk=0x2241c60) at ssl/ssl_cert.c:439
x = 0x223faf0
i = 0
verify_store = 0x223d220
ctx = 0x2254ea0
param = 0x22544f0
#11 0x00007f1e7f7f5c85 in tls_process_server_certificate (s=0x223e990, pkt=0x7ffc06898220) at ssl/statem/statem_clnt.c:1226
al = 0
i = 0
ret = 0
exp_idx = 0
cert_list_len = 1780
cert_len = 1777
x = 0x0
certstart = 0x2243f2a "0\202\006\355\060\202\006\234\240\003\002\001\002\002\020\001\321\361\201eE\277@"
certbytes = 0x224461b ""
sk = 0x2241c60
pkey = 0x0
#12 0x00007f1e7f7f4981 in ossl_statem_client_process_message (s=0x223e990, pkt=0x7ffc06898220) at ssl/statem/statem_clnt.c:624
st = 0x223e9d8
#13 0x00007f1e7f7f2f1b in read_state_machine (s=0x223e990) at ssl/statem/statem.c:589
st = 0x223e9d8
ret = 1
mt = 11
len = 1783
transition = 0x7f1e7f7f3e39 <ossl_statem_client_read_transition>
pkt = {curr = 0x224461b "", remaining = 0}
process_message = 0x7f1e7f7f48ef <ossl_statem_client_process_message>
post_process_message = 0x7f1e7f7f4a24 <ossl_statem_client_post_process_message>
max_message_size = 0x7f1e7f7f483a <ossl_statem_client_max_message_size>
cb = 0x0
#14 0x00007f1e7f7f29bc in state_machine (s=0x223e990, server=0) at ssl/statem/statem.c:385
buf = 0x0
Time = 1479452021
cb = 0x0
st = 0x223e9d8
ret = -1
ssret = 1
#15 0x00007f1e7f7f24b1 in ossl_statem_connect (s=0x223e990) at ssl/statem/statem.c:170
No locals.
#16 0x00007f1e7f7cd201 in ssl3_write_bytes (s=0x223e990, type=23, buf_=0x2228630, len=0) at ssl/record/rec_layer_s3.c:377
buf = 0x2228630 "x\222\v\177\036\177"
tot = 0
n = 0
split_send_fragment = 7400960
maxpipes = 0
max_send_fragment = 913408
nw = 3670016
u_len = 0
wb = 0x223ed68
i = 57344
#17 0x00007f1e7f7d9c1d in ssl3_write (s=0x223e990, buf=0x2228630, len=0) at ssl/s3_lib.c:3822
No locals.
#18 0x00007f1e7f7e6faa in SSL_write (s=0x223e990, buf=0x2228630, num=0) at ssl/ssl_lib.c:1605
No locals.
#19 0x000000000044f81d in s_client_main (argc=0, argv=0x7ffc06899040) at apps/s_client.c:2226
sbio = 0x2242570
key = 0x0
---Type <return> to continue, or q <return> to quit---
con = 0x223e990
ctx = 0x223caf0
chain = 0x0
cert = 0x0
vpm = 0x221ebb0
exc = 0x0
cctx = 0x221ec30
ssl_args = 0x0
dane_tlsa_domain = 0x0
dane_tlsa_rrset = 0x0
dane_ee_no_name = 0
crls = 0x0
meth = 0x7f1e7fa2ab40 <TLS_client_method_data.20660>
CApath = 0x0
CAfile = 0x7ffc0689985d "/usr/share/ca-certificates/extra/VipNet-CA.crt"
cbuf = 0x2228630 "x\222\v\177\036\177"
sbuf = 0x2232600 ""
mbuf = 0x2234610 ""
proxystr = 0x0
connectstr = 0x221ece0 "10.0.99.50:443"
cert_file = 0x0
key_file = 0x0
chain_file = 0x0
chCApath = 0x0
chCAfile = 0x0
host = 0x221ed00 "10.0.99.50"
port = 0x221ed20 "443"
inrand = 0x0
passarg = 0x0
pass = 0x0
vfyCApath = 0x0
vfyCAfile = 0x0
sess_in = 0x0
sess_out = 0x0
crl_file = 0x0
p = 0x7ffc06898754 ""
xmpphost = 0x0
ehlo = 0x47fcb5 "mail.example.com"
timeout = {tv_sec = 0, tv_usec = 0}
timeoutp = 0x0
readfds = {__fds_bits = {0 <repeats 16 times>}}
writefds = {__fds_bits = {8, 0 <repeats 15 times>}}
noCApath = 0
noCAfile = 0
build_chain = 0
cbuf_len = 0
cbuf_off = 0
cert_format = 32773
key_format = 32773
crlf = 0
full_log = 1
mbuf_len = 0
prexit = 0
sdebug = 0
reconnect = 0
verify = 0
vpmtouched = 0
ret = 1
in_init = 1
i = 1
nbio_test = 0
s = 3
k = 0
width = 4
state = 0
sbuf_len = 0
sbuf_off = 0
cmdletters = 1
socket_family = 0
socket_type = 1
starttls_proto = 0
crl_format = 32773
crl_download = 0
write_tty = 0
read_tty = 1
write_ssl = 1
read_ssl = 1
tty_on = 0
ssl_pending = 0
at_eof = 0
read_buf_len = 0
fallback_scsv = 0
randamt = 0
o = OPT_EOF
enable_timeouts = 0
socket_mtu = 0
ssl_client_engine = 0x0
e = 0x0
servername = 0x0
alpn_in = 0x0
tlsextcbp = {biodebug = 0x0, ack = 0}
ssl_config = 0x0
serverinfo_types = {256, 0, 0, 0, 36633, 67, 0, 0, 35456, 1673, 32764, 0, 27648, 105, 0, 0, 35296, 1673, 32764, 0, 25696, 32582, 32542, 0, 0, 0, 0, 0, 35328, 1673, 32764, 0, 35456, 1673, 32764, 0,
58480, 545, 0, 0, 0, 0, 7, 0, 60032, 545, 0, 0, 50480, 546, 0, 0, 32, 0, 0, 0, 46304, 2370, 0, 0, 36585, 67, 0, 0, 35360, 1673, 32764, 0, 24047, 32582, 32542, 0, 35456, 1673, 32764, 0, 58480, 545, 0,
0, 46304, 2370, 0, 0, 60032, 545, 0, 0, 27648, 105, 0, 0, 42240, 5516, 4135, 37986, 35392, 1673, 32764, 0}
serverinfo_count = 0
start = 0
len = 2127585272
next_proto_neg_in = 0x0
srppass = 0x0
srp_lateuser = 0
srp_arg = {srppassin = 0x0, srplogin = 0x0, msg = 0, debug = 0, amp = 0, strength = 1024}
ctlog_file = 0x0
ct_validation = 0
min_version = 0
max_version = 0
prot_opt = 0
no_prot_opt = 0
async = 0
split_send_fragment = 0
max_pipelines = 0
connect_type = use_inet
count4or6 = 0
c_nbio = 0
c_msg = 0
c_ign_eof = 0
---Type <return> to continue, or q <return> to quit---
c_brief = 0
c_tlsextdebug = 0
c_status_req = 0
bio_c_msg = 0x0
__PRETTY_FUNCTION__ = "s_client_main"
#20 0x0000000000438c2b in do_cmd (prog=0x221e470, argc=5, argv=0x7ffc06899040) at apps/openssl.c:471
f = {type = FT_none, name = 0x7ffc0689984c "s_client", func = 0x7ffc06898ab0, help = 0x43801c <lh_FUNCTION_retrieve+35>}
fp = 0x696c00 <functions+1152>
#21 0x000000000043835c in main (argc=5, argv=0x7ffc06899040) at apps/openssl.c:177
f = {type = 23, name = 0x6a2fe0 <prog> "s_client", func = 0x7f1e7ed05ff8, help = 0x7f1e7ecf8d80}
fp = 0x0
prog = 0x221e470
copied_argv = 0x0
p = 0x0
pname = 0x6a2fe0 <prog> "s_client"
buf = "t\213\211\006\374\177\000\000D\274\243\177\036\177\000\000@\320\304\177\036\177\000\000\036\005\000\000\000\000\000\000\350\344\304\177\036\177\000\000\200\215\317~\036\177\000\000\370_\320~\036\177\000\000{ģ\177\036\177\000\000\036\005\000\000\000\000\000\000\370_\320~\036\177\000\000\350\344\304\177\036\177\000\000\070\214\211\006\374\177\000\000\064\214\211\006\374\177\000\000\021\276\243\177\036\177\000\000\b\215\211\006\374\177\000\000j\377/\177\036\177\000\000P!.\177\036\177\000\000\070\214\211\006\374\177\000\000\256\207\377\000\000\000\000\202\035\376\003\000\000\000\000.\000\000\000\000\000\000\000D\274\243\177\036\177\000\000\370_\320~\036\177\000\000F\b\000\000\000\000\000\000\350\344\304\177\036\177\000\000\200"...
prompt = 0x7ffc06898b78 "{ģ\177\036\177"
arg = {size = 0, argc = 0, argv = 0x0}
first = 2143610088
n = 32542
i = 32764
ret = 0
This seems to be triggered by insufficient checks in gost_ec_verify function. sig_r and sig_s should be checked before calling BN_is_zero():
This patch prevents NULL pointer dereference, but still behaviour is unexpected.
openssl fails with 'certificate signature failure' although both server and CA certificates are valid and the command works OK when used with 'OpenSSL 1.0.2g' with gost engine bundled.