Code Monkey home page Code Monkey logo

csvjson's Introduction

csvjson

Simple CSV to JSON and JSON to CSV converter with stream support

for Node.js and Browser.

© Pradeep Mishra, Licensed under the MIT-LICENSE

Features

  • CSV to JSON object
  • CSV to schema JSON object
  • CSV to array object
  • CSV to column array object
  • JSON object to CSV
  • JSON array to CSV
  • Stream support

Example usage

var csvjson = require('csvjson');

/*
    sample.csv

    sr,name,age,gender
    1,rocky,33,male
    2,jacky,22,male
    3,suzy,21,female

*/


/*
    schema_sample.csv

    created,contact.name,contact.age+,contact.number+,address[],address[],contact.hobbies[],contact.hobbies[],-id
    2014-11-12,Pradeep,25,4352436,MG Road,Mumbai,pc games,guitar,5
    2014-10-06,Arnav,16,7364537,KB Road,Mumbai,pc games,traveling,7


*/


/*
schema_sample2.csv

name,age,contacts[0].name,contacts[0].phone,contacts[1].name,contacts[1].phone,musician,instruments.past,instruments.current[],instruments.current[]
Mark,33,Jim Palmer,8888888888,Marcus Aurelius,7309899877,Yes,Guitar,Drums,Bass Guitar
Jeff,27,John Doe,8009008000,Michael Corleone,2121001000,Yes,Drums,Flute,Trumpet

*/

/*
jsoncsv.json


{
 "book": {
   "person": [
     {
       "firstName": "Jane",
       "lastName": "Doe",
       "age": "25",
       "address": {
         "streetAddress": "21 2nd Street",
         "city": "Las Vegas",
         "state": "NV",
         "postalCode": "10021-3100"
       },
       "hobbies" : ["gaming", "volleyball"]
     },
     {
       "firstName": "Agatha",
       "lastName": "Doe",
       "age": "25",
       "address": {
         "streetAddress": "21 2nd Street",
         "city": "Las Vegas",
         "state": "NV",
         "postalCode": "10021-3100"
       },
       "hobbies" : ["dancing", "politics"]
     }
   ]
 }
}


*/

convert csv data to json object

var data = fs.readFileSync(path.join(__dirname, 'schema_sample2.csv'), { encoding : 'utf8'});
/*
{
    delimiter : <String> optional default is ","
    quote     : <String|Boolean> default is null
}
*/
var options = {
  delimiter : ',', // optional
  quote     : '"' // optional
};
// for multiple delimiter you can use regex pattern like this /[,|;]+/

/* 
  for importing headers from different source you can use headers property in options 
  var options = {
    headers : "sr,name,age,gender"
  };
*/

csvjson.toObject(data, options);

/*
    returns

    [
        {
            sr : 1,
            name : "rocky",
            age : 33,
            gender : "male"
        },
        {
            sr : 2,
            name : "jacky",
            age : 22,
            gender : "male"
        },
        {
            sr : 3,
            name : "suzy",
            age : 21,
            gender : "female"
        }

    ]

*/

convert csv data to schema json object

/*
    for creating schema of json object following key can be used in header of csv file:

    .   for defining nested json object
    []  for defining data as array (suffix)
        -- can add delimiter in the array (i.e. [;] for delimiter of ;)
        -- can nest objects in the array, index must be listed (i.e. [1] for index 1)
    +   for defining data as integer (suffix)
    -   for omitting data from result output (prefix)

*/


/*
    schema_sample.csv

    created,contact.name,contact.age+,contact.number+,address[],address[],contact.hobbies[;],-id,friends[0].name,friends[0].phone,friends[1].name,friends[1].phone
    2014-11-12,Pradeep,25,4352436,MG Road,Mumbai,pc games; guitar,5,Jeff,8761234567,Mike,1234567890
    2014-10-06,Arnav,16,7364537,KB Road,Mumbai,pc games; traveling,7,Steve,555555555,Pradeep,4352436

*/

var data = fs.readFileSync(path.join(__dirname, 'schema_sample.csv'), { encoding : 'utf8'});
/*
{
    delimiter : <String> optional default is ","
    quote     : <String|Boolean> default is null
}
*/
var options = {
  delimiter : ',', // optional
  quote     : '"' // optional
};

// for multiple delimiter you can use regex pattern like this /[,|;]+/

/* 
  for importing headers from different source you can use headers property in options 
  var options = {
    headers : "created,contact.name,contact.age+,contact.number+,address[],address[],contact.hobbies[;],-id,friends[0].name,friends[0].phone,friends[1].name,friends[1].phone"
  };
*/

csvjson.toSchemaObject(data, options)

/*
    returns

    [
        {
            "created":"2014-11-12",
            "contact":{
                "name":"Pradeep","
                age":25,
                "number":4352436,
                "hobbies":["pc games","guitar"]

            },
            "address":["MG Road","Mumbai"],
            "friends":[
                {
                    "name": "Jeff",
                    "phone": "8761234567"
                },
                {
                    "name": "Mike",
                    "phone": "1234567890"
                }
            ]
        },
        {
            "created":"2014-10-06",
            "contact":{"
                name":"Arnav",
                "age":16,
                "number":7364537,
                "hobbies":["pc games","traveling"]

            },
            "address":["KB Road","Mumbai"],
            "friends":[
                {
                    "name": "Steve",
                    "phone": "5555555555"
                },
                {
                    "name": "Pradeep",
                    "phone": "4352436"
                }
            ]
        }

    ]


*/

convert csv data to array object

var data = fs.readFileSync(path.join(__dirname, 'sample.csv'), { encoding : 'utf8'});

/*
{
    delimiter : <String> optional default is ","
    quote     : <String|Boolean> default is null
}
*/
var options = {
  delimiter : ',', // optional
  quote     : '"' // optional
};

// for multiple delimiter you can use regex pattern like this /[,|;]+/

csvjson.toArray(data, options);

/*
    returns
    [
        ["sr","name","age","gender"],
        ["1","rocky","33","male"],
        ["2","jacky","22","male"],
        ["3","suzy","21","female"]
    ]

*/

convert csv data to column array object

var data = fs.readFileSync(path.join(__dirname, 'sample.csv'), { encoding : 'utf8'});

/*
{
    delimiter : <String> optional default is ","
    quote     : <String|Boolean> default is null
}
*/
var options = {
  delimiter : ',', // optional
  quote     : '"' // optional
};

// for multiple delimiter you can use regex pattern like this /[,|;]+/

/* 
  for importing headers from different source you can use headers property in options 
  var options = {
    headers : "sr,name,age,gender"
  };
*/

csvjson.toColumnArray(data, options);

/*
    returns

    {
        sr: [ '1', '2', '3' ],
        name: [ 'rocky', 'jacky', 'suzy' ],
        age: [ '33', '22', '21' ],
        gender: [ 'male', 'male', 'female' ]
    }

*/

convert json object to csv data

var data = fs.readFileSync(path.join(__dirname, 'jsoncsv.json'), { encoding : 'utf8'});
var options = {
    delimiter   : ",",
    wrap        : false
}

/* supported options

    delimiter = <String> optional default value is ","
    wrap  = <String|Boolean> optional default value is false
    headers = <String> optional supported values are "full", "none", "relative", "key"
    objectDenote = <String> optional default value is "."
    arrayDenote = <String> optional default value is "[]"
*/


csvjson.toCSV(data, options);

/*
returns

book.person[].firstName,book.person[].lastName,book.person[].age,book.person[].address.streetAddress,book.person[].address.city,book.person[].address.state,book.person[].address.postalCode,book.person[].hobbies[]
Jane,Doe,25,21 2nd Street,Las Vegas,NV,10021-3100,gaming;volleyball
Agatha,Doe,25,21 2nd Street,Las Vegas,NV,10021-3100,dancing;politics

*/

convert csv to json using stream

var read = fs.createReadStream(path.join(__dirname, 'sample.csv'));
var write = fs.createWriteStream(path.join(__dirname, 'sample.json'));
var toObject = csvjson.stream.toObject();
var stringify = csvjson.stream.stringify();
read.pipe(toObject).pipe(stringify).pipe(write);

/*
following functions available for stream transformation
csvjson.stream.toObject()
csvjson.stream.toArray()
csvjson.stream.toColumnArray()
csvjson.stream.toSchemaObject()
*/

/*
csvjson.stream.stringify([space<number>])
stringify function accepts optional space parameter to format output
*/
npm install csvjson --save

csvjson's People

Contributors

karbassi avatar pradeep-mishra avatar ralucas avatar spiderpoul avatar

Stargazers

 avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar

Watchers

 avatar  avatar  avatar

csvjson's Issues

Line Breaks break Parser

If there are line breaks in a double quoted .csv field the parser splits the output on the line breaks as well as the commas resulting in really messed up objects.

Double quotes in CSV cause error

This is a duplicate of unsolved #12.

When running this code:

const csvjson = require('csvjson');

const options = {
  delimiter: ',',
  quote: true
};

const data = `one,two
"some data", "some ""other"" data"`;

const obj = csvjson.toObject(data, options);

console.log(obj);

the output is:

[ { one: 'some data', two: 'data' } ]

and should be:

[ { one: 'some data', two: 'some "other" data' } ]

This also occurs with the quote option set to '"'.

make it work in the browser

When using in the browser like this:

const pcsToImport = csvjson.toObject(data).output

I get this error:

Uncaught TypeError: fs.existsSync is not a function

If you don't want to make it work in the browser please state this in the readme.

Add documentation

JSON to CSV creates header rows with object and array notation. Great. I need to get rid of them.
The objectDenote and arrayDenote are optional and default to their respective defaults. But what is the option to get rid of them without modifying the file directly? No documentation for what the options are.

Filter exported fields

Hello,
Is there a way to filter which fields/keys to export when converting json to csv?

Error while compiling in an Angular application

I am trying to use csvjson in my Angular application. I did npm i csvjson --save

But I am getting the following error:

ERROR in ./node_modules/csvjson/src/stream.js Module not found: Error: Can't resolve 'stream' in 'D:\Projects\MyProject\node_modules\csvjson\src'

When I looked into this src folder in node_modules\csvjson\src -> I could find all 3 files app.js, helper.js and stream.js

We are really stuck on this. Please help...

Partial Header if '-' in key when converting JSON > CSV

When I'm converting JSON to CSV using the options:
var options = { headers : "key", delimiter : ";" }

One of the keys in my object is "Asset Sub-Type". The full CSV is rendered fine, but the header for the field associated with "Asset Sub-Type" is renamed to just "Type".

Support schema object generation of arrays of a objects

For example, it would be great to have

id,title,contents[].name,contents[].title,contents[].type

{"id": "XXX", "title": "YYY", "contents": [ { "name": "AAA", "title": "BBB", "type": "CCC"} ]

Similarly, something like "contents[1].title" and "contents[2].title" could be used for multiple objects within an array

Not able to change objectDenote and arrayDenote to empty strings.

Hoping to do what the title says. I am pulling in answers from a form, and then turning them from JSON into CSV. Sometimes the answers are arrays, sometimes not (multiple checkboxes). When that happens it adds the non-arrays and arrays as separate headers (for the same question).

For example, "question_2" and "question_2[]" become two headers, but I would just like "question_2" as the header. I changed the source code locally, and it worked when I removed the brackets from the string, but setting them as empty strings does not work.

@pradeep-mishra

TypeError: Cannot read property 'trim' of undefined

version 4.1.2

var data = "
A;B;C\n
;\n
"
csvjson.toObject(data, {delimiter:';'})

result:

TypeError: Cannot read property 'trim' of undefined
       at _trimQuote (./node_modules/csvjson/index.js:397:15)

expected:
not sure...

  • throw your own descriptive parse exception?
  • add callback / promise interface to pass back errors nicely?

Comma inside String error

I'm using csvjson in a React project.
When i try to use toColumnArray function with a CSV with a comma inside a string (eg "foo,bar"), i get the following error:
index.js:34 Uncaught TypeError: Cannot read property 'push' of undefined.
It seems the algorythim identifies comma as CSV separator instead part of the string

Allow configurable delimiter character

Please allow the character used as delimiter between CSV fields to be configured. Common characters are not only comma (,) but also semi-colon (;) and tab (/t)

This is making json data with a single quotation.

As u know JSON sometimes asks us strict rules which are that we have to use double quotations in JSON. In your examples, it looks fine but it generates only single quotations or no quotations. Would you check it once? I've tried it several times to get exact same result with your examples. I couldn't make it. If I'm doing wrong, leave a comment. Thx.

Wrong parsing if comma is in cells as well

Example:
JS =>

let sheet = (await axios.get(config.docs.technicalAnalysisIndicators)).data;
let result = csvjson.toObject(sheet, { encoding: 'utf8', delimiter: ',', quote: /['",]+/ });
return result;

CSV =>

func,name,shortname,caption,variable,predefined,min,max,step,values,type,category,groupCaption,description,notes
generic,Validation Size,generic,Validation Size,validation_size,0.2,0.1,0.5,0.1,,float,Machine Learning,It's a set of data used to access the strength and utility of a predictive relationship,,
generic,Seed,generic,Seed,seed,7,1,20,1,,integer,Machine Learning,description,,
generic,n Fold,generic,n Fold,num_fold,10,2,20,1,,integer,Machine Learning,description,,
generic,Scoring,generic,Scoring,scoring,accuracy,,,,"accuracy,recall",string,Machine Learning,"A ""score function"" or ""scoring rule"" measures the accuracy of probabilistic predictions",,Per ora l'utente non può selezionare nulla ma in futuro potremmo aggiungere l'opzione di scelta
LogisticRegression,Logistic Regression,LR,Regularization,lr_regularization,l2,,,,"l1, l2",string,Machine Learning,It's the go-to method for binary classification problems,penalty,
LogisticRegression,Logistic Regression,LR,C,lr_c,1.0,0.1,10.0,0.1,,float,Machine Learning,It's the go-to method for binary classification problems,C,
LogisticRegression,Logistic Regression,LR,Solver,lr_solver,liblinear,,,,"newton-cg, lbfgs, liblinear",string,Machine Learning,It's the go-to method for binary classification problems,solver,"Se seleziona lr_regularization = L1 allora il solver = Linear, se seleziona lr_regularization = L2 allora il solver = Linear, Newton-CG, L-BFGS"
LinearDiscriminantAnalysis,Linear Discriminant Analysis,LDA,Solver,lda_solver,svd,,,,"cvd, lsqr, eingen",string,Machine Learning,It's the go-to linear method for multiclass classification problems,solver,

Result is wrong, from "values" column everything gets messy =>

{
        "func": "generic",
        "name": "Scoring",
        "shortname": "generic",
        "caption": "Scoring",
        "variable": "scoring",
        "predefined": "accuracy",
        "min": "",
        "max": "",
        "step": "",
        "values": "\"accuracy",
        "type": "recall\"",
        "category": "string",
        "groupCaption": "Machine Learning",
        "description": "A \"\"score function\"\" or \"\"scoring rule\"\" measures the accuracy of probabilistic predictions",
        "notes": ""
    },

Support CSV header/schema in separate file/object/string

Not all CSV files have its header/schema in the same file as the CSV data itself. Please allow use of header/schema information from a different source, such a separate file, an object, or string. (either is fine for my immediate needs)

Module not found: Error: Can't resolve 'stream' in .../node_modules/csvjson/src'

./node_modules/csvjson/src/stream.js:17:15-32 - Error: Module not found: Error: Can't resolve 'stream' in .../node_modules/csvjson/src'
Did you mean './stream'?
Requests that should resolve in the current directory need to start with './'.
Requests that start with a name are treated as module requests and resolve within module directories

If changing the source code is not an option there is also a resolve options called 'preferRelative' which tries to resolve these kind of requests in the current directory too.

BREAKING CHANGE: webpack < 5 used to include polyfills for node.js core modules by default.
This is no longer the case. Verify if you need this module and configure a polyfill for it.

If you want to include a polyfill, you need to:
- add a fallback 'resolve.fallback: { "stream": require.resolve("stream-browserify") }'
- install 'stream-browserify'
If you don't want to include a polyfill, you can use an empty module like this:
resolve.fallback: { "stream": false }

using angular 14 and node 18.

Double quotes in CSV cause error

When parsing CSV, escaped double quotes cause error.
TypeError: Cannot read property '0' of null at node_modules/csvjson/index.js:74:55 at Array.forEach (native) at node_modules/csvjson/index.js:73:21 at Array.forEach (native) at Object.toObject (node_modules/csvjson/index.js:67:13)

Double quotes in CSV are escaped with double quotes, so it looks looks like this:
"this string contains ""double quotes"""
But using toObject to parse CSV files which includes such escaped quotes seems to invoke this error.

Breaks in streaming mode

When used in streaming mode with a reasonable size file, it seems to choke at regular points - I'm guessing its at chunk boundaries. Each time it stops part way through a record and inserts a closing and opening bracket ][ and then continues from the next record.

The csv file I'm using has about 5000 lines and the breaks occur at roughly every 560 lines.

Line lengths are variable but less than 150 characters.

There are 12 columns in the data.

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. 📊📈🎉

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google ❤️ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.