Code Monkey home page Code Monkey logo

jsone's Introduction

jsone

hex.pm version Build Status Code Coverage License: MIT

An Erlang library for encoding, decoding JSON data.

Features

  • Provides simple encode/decode function only
  • RFC7159-compliant
  • Supports UTF-8 encoded binary
  • Pure Erlang
  • Highly Efficient
    • Maybe one of the fastest JSON library (except those which are implemented in NIF)
    • Decode function is written in continuation-passing style(CPS)

QuickStart

# clone
$ git clone git://github.com/sile/jsone.git
$ cd jsone

# compile
$ make compile

# run tests
$ make eunit

# dialyze
$ make dialyze

# Erlang shell
$ make start
1> jsone:decode(<<"[1,2,3]">>).
[1,2,3]

Enable HiPE

If you want to use HiPE compiled version, please add following code to your rebar.config.

{overrides,
  [
    {override, jsone, [{erl_opts, [{d, 'ENABLE_HIPE'}, inline]}]}
  ]}.

or use native profile. The make command supports profile as well. For example:

$ make start profile=native

Usage Example

%% Decode
> jsone:decode(<<"[1,2,3]">>).
[1,2,3]

> jsone:decode(<<"{\"1\":2}">>).
#{<<"1">> => 2}

> jsone:decode(<<"{\"1\":2}">>, [{object_format, tuple}]). % tuple format
{[{<<"1">>, 2}]}

> jsone:decode(<<"{\"1\":2}">>, [{object_format, proplist}]). % proplist format
[{<<"1">>, 2}]

> jsone:try_decode(<<"[1,2,3] \"next value\"">>). % try_decode/1 returns remaining (unconsumed binary)
{ok,[1,2,3],<<" \"next value\"">>}

% error: raises exception
> jsone:decode(<<"1.x">>).
** exception error: bad argument
     in function  jsone_decode:number_fraction_part_rest/6
        called as jsone_decode:number_fraction_part_rest(<<"x">>,1,1,0,[],<<>>)
     in call from jsone:decode/1 (src/jsone.erl, line 71)

% error: returns {error, Reason}
> jsone:try_decode(<<"1.x">>).
{error,{badarg,[{jsone_decode,number_fraction_part_rest,
                              [<<"x">>,1,1,0,[],<<>>],
                              [{line,228}]}]}}


%% Encode
> jsone:encode([1,2,3]).
<<"[1,2,3]">>

> jsone:encode(#{<<"key">> => <<"value">>}).  % map format
> jsone:encode({[{<<"key">>, <<"value">>}]}). % tuple format
> jsone:encode([{<<"key">>, <<"value">>}]).  % proplist format
<<"{\"key\":\"value\"}">>

> jsone:encode(#{key => <<"value">>}). % atom key is allowed
<<"{\"key\":\"value\"}">>

% error: raises exception
> jsone:encode(#{123 => <<"value">>}). % non binary|atom key is not allowed
** exception error: bad argument
     in function  jsone_encode:object_members/3
        called as jsone_encode:object_members([{123,<<"value">>}],[],<<"{">>)
     in call from jsone:encode/1 (src/jsone.erl, line 97)

% error: returns {error, Reason}
> jsone:try_encode({[{123, <<"value">>}]}).
{error,{badarg,[{jsone_encode,object_members,
                              [[{123,<<"value">>}],[],<<"{">>],
                              [{line,138}]}]}}

% 'object_key_type' option allows non-string object key
> jsone:encode({[{123, <<"value">>}]}, [{object_key_type, scalar}]).
<<"{\"123\":\"value\"}">>

% 'undefined_as_null' option allows encoding atom undefined as null
> jsone:encode(undefined,[undefined_as_null]).
<<"null">>

%% Pretty Print
> Data = [true, #{<<"1">> => 2, <<"array">> => [[[[1]]], #{<<"ab">> => <<"cd">>}, [], #{}, false]}, null].
> io:format("~s\n", [jsone:encode(Data, [{indent, 2}, {space, 1}])]).
[
  true,
  {
    "1": 2,
    "array": [
      [
        [
          [
            1
          ]
        ]
      ],
      {
        "ab": "cd"
      },
      [],
      {},
      false
    ]
  },
  null
]
ok

%% Number Format
> jsone:encode(1). % integer
<<"1">>

> jsone:encode(1.23). % float
<<"1.22999999999999998224e+00">> % default: scientific notation

> jsone:encode(1.23, [{float_format, [{decimals, 4}]}]). % decimal notation
<<"1.2300">>

> jsone:encode(1.23, [{float_format, [{decimals, 4}, compact]}]). % compact decimal notation
<<"1.23">>

%% If you want to safely cast object keys to atoms, the `attempt_atom' option will help.
> jsone:decode(<<"{\"hello\": \"world\"}">>, [{keys, attempt_atom}]).
#{<<"hello">> => <<"world">>}  % There is no atom named "hello", so the key is decoded as binary.

> hello.  % Create "hello" atom.
hello

> jsone:decode(<<"{\"hello\": \"world\"}">>, [{keys, attempt_atom}]).
#{hello => <<"world">>} % Now, the key is decoded as atom.

Data Mapping (Erlang <=> JSON)

Erlang                  JSON             Erlang
=================================================================================================

null                   -> null                       -> null
undefined              -> null                       -> undefined                  % undefined_as_null
true                   -> true                       -> true
false                  -> false                      -> false
<<"abc">>              -> "abc"                      -> <<"abc">>
abc                    -> "abc"                      -> <<"abc">> % non-special atom is regarded as a binary
{{2010,1,1},{0,0,0}}   -> "2010-01-01T00:00:00Z"     -> <<"2010-01-01T00:00:00Z">>     % datetime*
{{2010,1,1},{0,0,0.0}} -> "2010-01-01T00:00:00.000Z" -> <<"2010-01-01T00:00:00.000Z">> % datetime*
123                    -> 123                        -> 123
123.4                  -> 123.4                      -> 123.4
[1,2,3]                -> [1,2,3]                    -> [1,2,3]
{[]}                   -> {}                         -> {[]}                       % object_format=tuple
{[{key, <<"val">>}]}   -> {"key":"val"}              -> {[{<<"key">>, <<"val">>}]} % object_format=tuple
[{}]                   -> {}                         -> [{}]                       % object_format=proplist
[{<<"key">>, val}]     -> {"key":"val"}              -> [{<<"key">>, <<"val">>}]   % object_format=proplist
#{}                    -> {}                         -> #{}                        % object_format=map
#{key => val}          -> {"key":"val"}              -> #{<<"key">> => <<"val">>}  % object_format=map
{{json, IOList}}       -> Value                      -> ~~~                        % UTF-8 encoded term**
{{json_utf8, Chars}}   -> Value                      -> ~~~                        % Unicode code points**

* see jsone:datetime_encode_format()

** {json, IOList} and {json_utf8, Chars} allows inline already encoded JSON values. For example, you obtain JSON encoded data from database so you don't have to decode it first and encode again. See jsone:json_term().

API

See EDoc Document

Benchmark

The results of poison benchmarking.

See the benchmark/README.md file for more information.

Encoding (Unit: IPS=inputs per second)

Input data \ Library Jason jiffy JSON* jsone JSX Poison Tiny
Blockchain 2.77 K 4.55 K 0.45 K 1.44 K (3) 0.60 K 1.30 K 0.99 K
Giphy 230.65 487.67 47.73 114.57 (4) 44.97 114.57 113.59
GitHub 880.03 1566.67 139.79 300.26 (5) 99.68 424.75 455.07
GovTrack 6.57 24.92 2.33 5.35 (5) 2.65 7.06 7.86
Issue 90 22.80 21.92 0.77 14.30 (3) 5.33 12.60 12.95
JSON Generateor 200.40 606.81 42.45 147.12 (4) 68.73 187.95 123.93
Pokedex 209.51 776.67 62.60 161.45 (4) 69.87 190.93 125.16
UTF-8 unescaped 626.25 6644.53 1167.89 582.41 (4) 273.48 401.44 220.14

* Only JSON didn't escape non-ASCII unicode characters on the encoding

Decoding (Unit: IPS=inputs per second)

Input data \ Library Jason jiffy JSON jsone JSX Poison Tiny
Blockchain 2.75 K 2.62 K 0.35 K 2.21 K (3) 0.89 K 1.32 K 1.49 K
Giphy 212.18 243.45 35.67 109.11 (5) 64.32 110.76 114.54
GitHub 973.41 1052.94 137.02 662.39 (3) 271.97 438.79 542.67
GovTrack 10.77 8.32 0.80 5.08 (3) 2.81 3.58 3.65
Issue 90 17.85 41.16 0.88 10.79 (5) 6.02 13.63 14.03
JSON Generateor 320.79 243.93 25.16 184.23 (3) 111.24 135.47 139.78
JSON Generateor (Pretty) 273.57 205.09 25.04 158.82 (3) 97.93 123.31 136.65
Pokedex 527.63 285.43 33.70 245.36 (3) 140.90 172.45 152.59
UTF-8 escaped 1224.48 7923.08 326.43 573.70 (4) 550.36 918.21 520.31
UTF-8 unescaped 5.56 K 12.54 K 1.35 K 5.09 K (3) 3.30 K 4.39 K 1.46 K

License

This library is released under the MIT License.

See the COPYING file for full license information.

jsone's People

Contributors

alberdi avatar benoitc avatar bkolodziej avatar flameeyes avatar jesperes avatar kianmeng avatar kuenishi avatar kusano avatar licenser avatar michalmuskala avatar pichi avatar russelldb avatar sile avatar spinute avatar srenatus avatar zmstone avatar zzydxm avatar

Stargazers

 avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar

Watchers

 avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar

jsone's Issues

jsone fails with encode after decode json

B = jsone:decode(<<" {\"id\":\"¾H^Z�y^GID[zÓ� ^VC£�æ\"} ">>), jsone:encode(B),

{badarg, [{jsone_encode,escape_string, [<<190,72,94,90,158,121,94,71,73,68,91,122,211,136,32,32, 94,86,67,163,147,230>>, [{object_members,[]}], <<"{\"id\":\"">>, {encode_opt_v2,false,false, [{scientific,20}], {iso8601,0}, string,0,0,false}], [{line,262}]},

Streaming decode

Hi!

Parsing a document incrementally, feeding the decoder chunks as they arrive as TCP packets over the socket, could speed up the total handling of a request.

We're receiving ~2MB JSON requests over HTTP/2 where each document is interleaved with other requests in the same HTTP/2 connection, which makes the latency for per request even higher when there are many concurrent requests on the same connection.

An idea is to use an API similar to how JSX streming decode API:

1> {incomplete, F} = jsx:decode(<<"[">>, [stream]).
{incomplete,#Fun<jsx_decoder.1.122947756>}
2> F(end_stream).  % can also be `F(end_json)`
** exception error: bad argument
3> {incomplete, G} = F(<<"]">>).
{incomplete,#Fun<jsx_decoder.1.122947756>}
4> G(end_stream).  % can also be `G(end_json)`
[]

We could use another representation too. Any preference? Would you be willing to accept a PR?

encode datatime

jsx:encode([{<<"create_date">>,{{2015,6,25},{14,57,25}}}]) 
=> <<"{\"create_date\":\"2015-06-25T14:57:25Z\"}">>

jsone:encode([{<<"create_date">>,{{2015,6,25},{14,57,25}}}])
=> ** exception error: bad argument

jsone fails to compile when using Erlang 21.

I'm not too sure which one of my dependencies is relying on jsone (Might be the version required by the latest xprof as it compiles when removing xprof from my deps) - I noticed that it cannot be compiled using Erlang OTP 21 as part of an Elixir project. Tried using both Elixir 1.6.4, 1.6.5 and 1.6.6.

Compilation error:

===> Compiling jsone
===> Compiling src/jsone.erl failed
src/jsone.erl:261: erlang:get_stacktrace/0: deprecated; use the new try/catch syntax for retrieving the stack backtrace
src/jsone.erl:310: erlang:get_stacktrace/0: deprecated; use the new try/catch syntax for retrieving the stack backtrace

** (Mix) Could not compile dependency :jsone, "/home/doodloo/.mix/rebar3 bare compile --paths "/home/doodloo/Documents/Professional/TML/linky-api/_build/dev/lib/*/ebin"" command failed. You can recompile this dependency with "mix deps.compile jsone", update it with "mix deps.update jsone" or clean it with "mix deps.clean jsone"

Why is rebar3 included?

It's very strange to me that the rebar3 binary is included in the repo.
Travis already supports rebar3 without including it manually and it's very easy to get hold of as a developer.
Can it be removed?

J /= jsone:decode(jsone:encode(J).

hi,

i thought that

J=jsone:decode(jsone:encode(J).

should hold true for any encodeble J but this not true as you can see from

J=[{a,1},{b,2}]. % which is a list of tuples

jsone:encode(J).
<<"{"a":1,"b":2}">>

jsone:decode(jsone:encode(J)).
#{<<"a">> => 1,<<"b">> => 2} % which is a map

is there anything i have missed?

e.

decoding invalid JSON starting with number and followed by extra characters doesn't throw exception

This is on version 1.5.6 as well as master branch.

~/git/jsone % make start
===> Verifying dependencies...
===> Analyzing applications...
===> Compiling jsone
===> Verifying dependencies...
===> Analyzing applications...
===> Compiling jsone
Erlang/OTP 24 [erts-12.1.2] [source] [64-bit] [smp:12:12] [ds:12:12:10] [async-threads:1] [jit] [dtrace]

Eshell V12.1.2  (abort with ^G)
1> ===> Booted jsone

1> jsone:decode(<<"1@">>).
1
2> jsone:decode(<<"1@s">>).
1

Convert map to json and back

Hey guys,
thx for providing this great lib. :-)

I have a question for the usage of your lib. I try to convert a map to json and back. According to the Data Mapping provided in your doc, this is the case:
#{key => val} -> {"key":"val"} -> #{<<"key">> => <<"val">>} % object_format=map

This is what I am doing:
1> Json = jsone:encode(#{key => val}).
<<"{"key":"val"}">>
2> Map = jsone:decode(Json).
{[{<<"key">>,<<"val">>}]}
3> maps:find(key, Map).

This is my result:
** exception error: {badmap,{[{<<"key">>,<<"val">>}]}}

And this is what I expect as result:
val

Can anybody provide me help on how to map a map to a json and back?

New line encoded chars are not decoded properly

Try to decode the following

<<"{\"to\":\"Jon Doe\",\"text\":\"Hello John,\n\nVisit me to renew.\"}">>

Error:

exception error: bad argument
     in function  jsone_decode:string/6
        called as jsone_decode:string(<<"\n\nVisit me to renew.\"}">>,
                                      <<"Hello John,\n\nVisit me to renew.\"}">>,
                                      0,
                                      [{object_next,<<"text">>,
                                        [{<<"to">>,<<"Jon Doe">>}]}],
                                      <<>>,
                                      {decode_opt_v2,proplist,false,binary,
                                       false})
     in call from jsone:decode/2 (src/jsone.erl, line 293)

encoding bug?

I've a test that automatically generate some values and I got the following error:

1> jsone:encode(#{<<"id">> => <<"aaaaaa">>,<<"v">> => <<128>>}).
** exception error: bad argument
     in function  jsone_encode:escape_string/4
        called as jsone_encode:escape_string(<<128>>,
                                             [{object_members,[]}],
                                             <<"{\"id\":\"aaaaaa\",\"v\":\"">>,
                                             {encode_opt_v2,false,false,
                                                            [{scientific,20}],
                                                            {iso8601,0},
                                                            string,0,0,false})

i'm not sure at this point if it's expected or not. Let me know :)

How to get jsone to work without the make script?

Eshell V8.2.1 (abort with ^G) 1> jsone:decode(<<"[1,2,3]">>). ** exception error: undefined function jsone:decode/1

When I use the make start it works fine, what do I need to setup so it works with erl?

Cannot build on Erlang R21

===> Compiling _build/default/lib/jsone/src/jsone.erl failed
_build/default/lib/jsone/src/jsone.erl:294: erlang:get_stacktrace/0: deprecated; use the new try/catch syntax for retrieving the stack backtrace
_build/default/lib/jsone/src/jsone.erl:343: erlang:get_stacktrace/0: deprecated; use the new try/catch syntax for retrieving the stack backtrace

Better errors than badarg

Is there any plan to add better errors than just badarg? It's problematic when trying to encode/decode inside more complex code:

    try
        complex_code(Binary) % Code that calls jsone:decode somewhere
    catch
        error:badarg:ST ->
            case ST of
                [{jsone_decode,_,_,_}|_] ->
                        ?LOG_WARNING("Bad JSON: ~p", [Binary]);
                _ ->
                    erlang:raise(error, badarg, ST)
            end
    end.

It's impossible to know where the badarg error comes from without checking the stack trace.

I apologize if this have been considered already, since there might be a lot of code changes / possible overhead (e.g. if try-catch is used).

Doesn't encode properly float values.

Hello,

I have an issue with jsone library and float numbers after migrating from jsonx:

jsone:encode([{<<"ss">>, 1.485}]).

will result in :

<<"{\"ss\":1.48500000000000009770e+00}">>

instead:

<<"{\"ss\":1.485}">>

Silviu

jsone fails to compile when setting overrides to enable HiPE

In the README's instructions for compiling jsone with HiPE, it says:

If you want to use HiPE compiled version, please add following code to your rebar.config.

{overrides,
  [
    {override, jsone, [{erl_opts, [{d, 'ENABLE_HIPE'}, inline]}]}
  ]}.

However, this has the unfortunate side effect of erasing the existing erl_opts in the rebar.config, including the platform_define settings, which make it very likely that the compilation will fail on pre-OTP 21 versions of Erlang.

Can the README be updated to provide instructions for how to enable HiPE compilation in the context of jsone as a dependency?

Note: when I filed this issue, I did not realize that it was because of my overrides, so the problem statement was incomplete. Here is the text of the original issue (for context):
As of the latest changes, it seems that jsone is no longer compatible with compiling on OTP 20 (and probably older releases as well) when it is added as a dependency:

./rebar3 compile
===> Verifying dependencies...
===> Compiling jsone
===> Compiling _build/default/lib/jsone/src/jsone.erl failed
_build/default/lib/jsone/src/jsone.erl:304: illegal pattern
_build/default/lib/jsone/src/jsone.erl:305: variable 'Reason' is unbound
_build/default/lib/jsone/src/jsone.erl:305: variable 'StackItem' is unbound
_build/default/lib/jsone/src/jsone.erl:305: variable '__StackTrace' is unbound
_build/default/lib/jsone/src/jsone.erl:353: illegal pattern
_build/default/lib/jsone/src/jsone.erl:354: variable 'Reason' is unbound
_build/default/lib/jsone/src/jsone.erl:354: variable 'StackItem' is unbound
_build/default/lib/jsone/src/jsone.erl:354: variable '__StackTrace' is unbound

This seems related to 0e42866. I am not sure what happened. It looks like it is written right, and it compiles when used directly (rebar3 compile from inside the jsone folder itself), but when it's a dependency of my project, it fails.

escape characters

jsone:encode(#{<<"key">> => <<"aaaa/bbbb">>}) => <<"{"key":"aaaa\/bbbb"}">>
jsx/jiffy:encode(#{<<"key">> => <<"aaaa/bbbb">>}) =><<"{"key":"aaaa/bbbb"}">>

when I use file:write_file, jsone result will be {"key":"aaaa/bbbb"}
others are right

Encoding issues

Hello,

The following code:

P = [{<<"operatorname">>,<<"TELEFÓNICA MÓVILES ESPAÑA">>},{<<"originaloperatorname">>,<<"TELEFÓNICA MÓVILES ESPAÑA">>}],
    jsone:encode(P, [{float_format, [{decimals, 4}, compact]}]).

Throw the following error:

** exception error: bad argument
     in function  jsone_encode:escape_string/4
        called as jsone_encode:escape_string(<<"�NICA M�VILES ESPA�A">>,
                                             [{object_members,
                                               [{<<"originaloperatorname">>,
                                                 <<"TELEF�NICA M�VILES ESPA�A">>}]}],
                                             <<"{\"operatorname\":\"TELEF">>,
                                             {encode_opt_v2,false,false,
                                              false,
                                              [{decimals,4},compact],
                                              {iso8601,0},
                                              string,0,0,false,false,
                                              fun jsone:term_to_json_string/1})

Sensible default handling for tuples

Consider the following

1> Tuple = {{key, value}, { key2, {key3, [value1, 2,3]}}}.
{{key,value},{key2,{key3,[value1,2,3]}}}

Produces:

2> jsone:encode(Tuple).
** exception error: bad argument
in function jsone_encode:value/4
called as jsone_encode:value({{key,value},{key2,{key3,[value1,2,3]}}},
[],<<>>,
{encode_opt_v2,false,false,false,
[{scientific,20}],
{iso8601,0},
string,0,0,false,false,undefined})
in call from jsone:encode/2 (jsone.erl, line 382)

I think we should have a sensible default in these cases. Here is one suggestion:

{"key":"value", "key2, {"key3":["value1", 2,3]}}

How would one be able to pass on hints on how to handle certain conversion when default fails?

escaping issues.

Hello,

First I'm not sure this is a bug in the library. But I will appreciated help from somebody on this issues I'm facing.
I have a proplists in Original and I'm expecting the encoding to produce Expected binary. But I have no clue what to do to
generate this output.

When I encode the proplists the \ char from the unicode code are double encoded and everything gets broken.

-module(test).

-export([t/0]).


t() ->
    Original = [
        {<<"s1">>, <<"\\u001b test 1 2 3">>}
    ],

    Expected = <<"{\"s1\":\"\\u001b test 1 2 3\"}">>,
    Encoded = encode(Original),

    io:format("original:~p~n", [Original]),
    io:format("encoded:~p~n", [Encoded]),
    io:format("decoded:~p~n", [decode(Encoded)]),
    io:format("expected:~p~n", [decode(Expected)]).

decode(V) ->
    jsone:decode(V, [{object_format, proplist}, {allow_ctrl_chars, true}]).

encode(V) ->
    jsone:encode(V, [{float_format, [{decimals, 4}, compact]}]).

Output:

original:[{<<"s1">>,<<"\\u001b test 1 2 3">>}]
encoded:<<"{\"s1\":\"\\\\u001b test 1 2 3\"}">>
decoded:[{<<"s1">>,<<"\\u001b test 1 2 3">>}]
expected:[{<<"s1">>,<<"\e test 1 2 3">>}]

Kind regards,
Silviu

Datetime is not allowed as head of array

1> jsone:encode([{{2015,6,25},{14,57,25}}]).       
** exception error: bad argument
     in function  jsone_encode:object_key/4
        called as jsone_encode:object_key({2015,6,25},
                                          [{object_value,{14,57,25},[]}],
                                          <<"{">>,
                                          {encode_opt_v2,false,
                                                         [{scientific,20}],
                                                         {iso8601,0},
                                                         string,0,0})
     in call from jsone:encode/2 (/home/hynek/work/altworx/jsone/_build/default/lib/jsone/src/jsone.erl, line 306)

Improve Benchmarks

It would be helpful to test more when doing the benchmarks. Testing with https://github.com/devinus/poison would provide more comprehensive results. The performance results without HiPE are more interesting than the results with HiPE, since HiPE is normally not used in production due to stability/bug issues.

badarg on probably valid term (encode)

erlang release: OTP17
erts:
jsone: 1054adc

input

T = [{formId,<<>>},{procId,<<83,54,68,68,70,85,82,78,55,67,50,49,79,76,67,66,67,74,49,79>>},{model,{[{<<112,114,111,99>>,<<111,107>>},{<<110,97,109,101>>,<<76,117,99,117,115>>},{<<105,115,65,117,116,104>>,<<116,114,117,101>>},{<<117,115,101,114,73,100>>,<<102,52,99,102,56,98,102,100,45,48,98,54,51,45,52,100,98,55,45,97,50,98,98,45,101,102,98,99,98,102,48,101,100,55,52,54>>}]}},{chatId,<<115,101,110,100,101,114>>},{class,<<46,103,101,116,85,115,101,114,73,110,102,111,46,115,101,110,100,101,114>>},{viewType,<<106,115,111,110>>},{viewModel,{[]}},{linkId,<<57,101,100,100,98,98,101,100,45,49,99,49,56,45,56,49,55,101,45,48,97,57,51,45,101,97,99,101,49,100,48,48,51,101,55,56>>},{created,1425304575552},{packetId,<<57,101,100,100,98,98,101,100,45,49,99,49,56,45,56,49,55,101,45,48,97,57,51,45,101,97,99,101,49,100,48,48,51,101,55,56>>},{from,<<115,101,110,100,101,114>>},{companyId,<<115,101,110,100,101,114>>},{robotId,<<103,101,116,85,115,101,114,73,110,102,111>>},{status,<<110,101,119>>}]
jsone:encode(T)

Exception

Ranch listener dispatchConv had connection process started with cowboy_protocol:start_link/4 at <0.4789.0> exit with reason: {[{reason,badarg},{mfa,{conv_base_handler,handle,2}},{stacktrace,[{jsone_encode,escape_string,[<<".sender">>,[{object_members,[{viewType,<<"json">>},{viewModel,{[]}},{linkId,<<"9eddbbed-1c18-817e-0a93-eace1d003e78">>},{created,1425304575552},{packetId,<<"9eddbbed-1c18-817e-0a93-eace1d003e78">>},{from,<<"sender">>},{companyId,<<"sender">>},{robotId,<<"getUserInfo">>},{status,<<"new">>}]}],<<123,34,102,111,114,109,73,100,34,58,34,34,44,34,112,114,111,99,73,100,34,58,34,83,54,68,68,70,85,82,78,55,67,50,49,79,76,67,66,67,74,49,79,34,44,34,109,111,100,101,108,34,58,123,34,112,114,111,99,34,58,34,111,107,34,44,34,110,97,109,101,34,58,34,76,117,99,117,115,34,44,34,105,115,65,117,116,104,34,58,34,116,114,117,101,34,44,34,117,115,101,114,73,100,34,58,34,102,52,99,102,56,98,102,100,45,48,98,54,51,45,52,100,98,55,45,97,50,98,98,45,101,102,98,99,98,102,48,101,100,55,52,54,34,125,44,34,99,104,97,116,73,100,34,58,34,115,101,110,100,101,114,34,44,34,99,108,97,115,115,34,58,34,46,103,101,116,85,0,0,0,0,0,0,0>>],[{line,144}]}

I cant reproduce this issue with erl -pa deps/jsone/ebin and typing input as well as attaching to the target node where issue appeared and typing input there.

How to prevent "/" from being escaped on encode

Even using native_utf8 option, I can't seem to prevent the encoder from escaping the forward slash.

Example:

Map = #{<<"K">> => #{<<"K1">> => <<"V1">>, <<"K2">> => <<"1/2">>}},
jsone:encode(Map, [native_utf8]).

Encoding produced for K2 is \"1\\/2\".

Encoding Strings

Hello, I could use some help.

I am trying to encode some data in Erlang but I am having trouble getting the patterns down.

Here's the pre encoding section section of my code

    ...
    Array = [],
    
    {Type, Priority, _, _, From, To, Msg, Data} = Event,
    
    %% Type => atom
    %% Priority => integer
    %% From => atom
    %% To => atom
    %% Msg => string
    %% Data => List of tuples: [..., {Key, Value}, ...]
    %%          where Key => atom
    %%                Value => integer or atom

    % create a json encoded instance
    
    EncodedData = lists:map(fun({Key, Value}) -> #{Key => Value}  end, Data),

    Instance = [
        {<<"type">>, Type},
        {<<"priority">>, Priority},
        {<<"from">>, From},
        {<<"to">>, To},
        {<<"msg">>, Msg},
        {<<"data">>, Data}
    ],
    
    EncodedArray = [Instance | Array],
    
    DataJSON = jsone:encode(EncodedArray),
    
    %% send DataJSON with cowboy 
    ...

Viewing in the browser, I get my output fine except for the msg whose value comes as an array of numbers

image

Please what changes am i to make such that i get a string instead of an array of numbers for the msg key?

Option to decode null to undefined

With undefined_as_null there is a situation where the encoded and decoded data does not match at this point as null is always decoded as null. It would be nice if the decode took undefined_as_null as an option as well to mirror the behavior of the encoder.

json decode exception : if value contains white space

jsone:decode(<<"{\"_comment_Car_Card\" : \"CONTENTSDB\"}">>). => ok.

jsone:decode(<<"{\"_comment_Car_Card\" : \"\tCONTENTSDB\"}">>). => error. (value contains \t )

** exception error: no function clause matching jsone_decode:string(<<"\tCONTENTSDB\"}">>,<<"\tCONTENTSDB\"}">>,0,
                                                                    [{object_next,<<"_comment_Car_Card">>,[]}],
                                                                    <<>>,
                                                                    {decode_opt_v1,map}) (src/jsone_decode.erl, line 139)
     in function  jsone:decode/2 (src/jsone.erl, line 228)

The expected value is [{<<"_comment_Car_Card">>,<<"\tCONTENTSDB">>}] .

Encoding of undefined atom as null

undefined is a default value for eg. not set fields in records.
undefined is accepted standard for null in Erlang.

Encoding undefined as text "undefined" is problematic when converting mnesia records to maps or list of tuples. This behaviour requires additional steps (or redefining all records to set null as default value instead of undefined).

My proposition is to add following line to jsone_encode module:

value(undefined, Nexts, Buf, Opt) -> next(Nexts, <<Buf/binary, "null">>, Opt);

...or probably better option at this point:

value(undefined, Nexts, Buf, Opt = ?OPT{undefined_as_null = true}) -> next(Nexts, <<Buf/binary, "null">>, Opt);

and extend encode_opt_v2:

-record(encode_opt_v2, {
          native_utf8 = false :: boolean(),
          float_format = [{scientific, 20}] :: [jsone:float_format_option()],
          datetime_format = {iso8601, 0} :: {jsone:datetime_format(), jsone:utc_offset_seconds()},
          object_key_type = string :: string | scalar | value,
          space = 0 :: non_neg_integer(),
          indent = 0 :: non_neg_integer(),
          undefined_as_null = false :: bolean()
         }).

build error on 0TP 21: Erlang:get_stracktrace is deprecated

When trying to build sone on OTP 21 you get the the following errors:

_build/default/lib/jsone/src/jsone.erl:294: erlang:get_stacktrace/0: deprecated; use the new try/catch syntax for retrieving the stack backtrace
_build/default/lib/jsone/src/jsone.erl:343: erlang:get_stacktrace/0: deprecated; use the new try/catch syntax for retrieving the stack backtrace

encoding fails with proplists with string keys

It is ok with binary keys

([email protected])42> jsone:encode([{<<"a">>, 1}]).                    
<<"{\"a\":1}">>

It fails with string keys

([email protected])43> jsone:encode([{"a", 1}]).    
** exception error: bad argument
     in function  jsone_encode:object_key/4
        called as jsone_encode:object_key("a",
                                          [{object_value,1,[]}],
                                          <<"{">>,
                                          {encode_opt_v2,false,false,false,
                                              [{scientific,20}],
                                              {iso8601,0},
                                              string,0,0,false,false,
                                              fun jsone:term_to_json_string/1})
     in call from jsone:encode/2 (src/jsone.erl, line 386)

Encoding crashes

Command executed:

using latest jsone master branch

 jsone:encode( [{severity,<<"debug">>},
             {datetime,<<"2019-07-25 16:26:19.581">>},
             {timestamp,1564052179581},
             {message,<<"p1_mysql_auth send packet 3: <<\"5ç\">>">>},
             {node,<<"ejabberd@pankaj-macbook">>},
             {pid,<<"<0.1398.0>">>},
             {file,<<"src/ejabberd_sql.erl">>},
             {module,<<"ejabberd_sql">>},
             {function,<<"log">>},
             {line,675}]).

error log

** exception error: bad argument
     in function  jsone_encode:escape_string/4
        called as jsone_encode:escape_string(<<"ç\">>">>,
                                             [{object_members,[{node,<<"ejabberd@pankaj-macbook">>},
                                                               {pid,<<"<0.1398.0>">>},
                                                               {file,<<"src/ejabberd_sql.erl">>},
                                                               {module,<<"ejabberd_sql">>},
                                                               {function,<<"log">>},
                                                               {line,675}]}],
                                             <<"{\"severity\":\"debug\",\"datetime\":\"2019-07-25 16:26:19.581\",\"timestamp\":1564052179581,\"message\":\"p1_mysql_a"...>>,
                                             {encode_opt_v2,false,false,false,
                                                            [{scientific,20}],
                                                            {iso8601,0},
                                                            string,0,0,false})
     in call from jsone:encode/2 (src/jsone.erl, line 360)

Handling of ASCII control characters in strings (encode)

Hi,

I am using jsone to output JSON in API consumed by various clients. One of issues I had was that following output:

jsone:encode(<<"\x01">>).
<<34,1,34>>

caused issues for python clients (unless they used json.loads(data, strict=False)).

I have added following lines:

escape_string(<<0:4, C:4, Str/binary>>, Nexts, Buf, Opt) -> escape_string(Str, Nexts, <<Buf/binary, $\\, $u, $0, $0, ?H8(C)>>, Opt);
escape_string(<<1:4, C:4, Str/binary>>, Nexts, Buf, Opt) -> escape_string(Str, Nexts, <<Buf/binary, $\\, $u, $0, $0, ?H8(C + 16)>>, Opt);

before

escape_string(<<0:1, C:7, Str/binary>>, Nexts, Buf, Opt) -> escape_string(Str, Nexts, <<Buf/binary, C>>, Opt);

which causes the output for control characters to change to something that can be consumed by json.loads(data) without disabling strict mode:

jsone:encode(<<"\x01">>).
<<"\"\\u0001\"">>

On a side note, I have also added:

value({struct,Value}, Nexts, Buf, Opt) -> object(Value, Nexts, Buf, Opt);

to jsone_encode for easier migration from mochijson2.

Please consider adding above lines to jsone.

Running tests with Erlang 20

Test suite fails to compile:

===> Compiling /pathto/git/eban/jsone/_build/test/lib/jsone/test/jsone_decode_tests.erl failed
/pathto/git/eban/jsone/_build/test/lib/jsone/test/jsone_decode_tests.erl:235: key <<"1">> will be overridden in expression

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. 📊📈🎉

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google ❤️ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.