This package combines redis cache and multiple postgres database together to contain a large(infinite) tables spanning across DBs,with following features
- Row count partitioning.
- Search with inbuilt time ranges.
- Self Id generation or User provided Ids for primary key(user provided id limited to 20000000).
- Custom schema defination.
- Custom indexes.
- Normal CRUD operations.
- Paginated results for all calls.
- Install using
npm i pg-infinity-table - Require in your project.
const infinityType = require('pg-infinity-table'); - Run postgres as local docker if required.
docker run --name pg-12.4 -e POSTGRES_PASSWORD=mysecretpassword -e POSTGRES_DB=pg-infinity-meta -p 5432:5432 -d postgres:12.4-alpine - Run redis as local docker if required.
docker run -p 6379:6379 -itd --rm redis:latest - Create 2 worker database for table across DB Eg: Infinity-1 & Infinity-2
- Instantiate with a postgres readers and writers and connections to redis.
- All done, Start using it!!.
- Initialize
const infinityTableType = require('pg-infinity-table');
const metaRedisConectionString = "redis://127.0.0.1:6379/";
const metaPGConectionString = "postgres://postgres:mysecretpassword@localhost:5432/pg-infinity-meta?application_name=perf-test";
const readConfigParams = {
connectionString: metaPGConectionString,
application_name: "Infinity Test",
max: 4 //4 readers
};
const writeConfigParams = {
connectionString: metaPGConectionString,
application_name: "Infinity Test",
max: 2 //2 Writer
};
const infinityDatabase = await infinityTableType(metaRedisConectionString, readConfigParams, writeConfigParams)- Add Resources
const maxTablesPerResource = 1000; //Maximum Tables that will be used to create infinity tables.
const maxRowsPerTable = 100; //<-Row Partitioning
const readConfigParamsDB1 = {
connectionString: "postgres://postgres:@localhost:5432/Infinity-1",
application_name: "e2e Test",
max: 4 //4 readers
};
const writeConfigParamsDB1 = {
connectionString: "postgres://postgres:@localhost:5432/Infinity-1",
application_name: "e2e Test",
max: 2 //2 Writer
};
const readConfigParamsDB2 = {
connectionString: "postgres://postgres:@localhost:5432/Infinity-2",
application_name: "e2e Test",
max: 4 //4 readers
};
const writeConfigParamsDB2 = {
connectionString: "postgres://postgres:@localhost:5432/Infinity-2",
application_name: "e2e Test",
max: 2 //2 Writer
};
let resourceId = await infinityDatabase.registerResource(readConfigParamsDB1, writeConfigParamsDB1, maxTablesPerResource, maxRowsPerTable);
console.log("Resource Id:" + resourceId);
resourceId = await infinityDatabase.registerResource(readConfigParamsDB2, writeConfigParamsDB2, maxTablesPerResource, maxRowsPerTable);
console.log("Resource Id:" + resourceId);- Create Infinity Table
const TableDefinition = [{
"name": "time",
"datatype": "bigint"
},
{
"name": "tagid",
"datatype": "integer"
},
{
"name": "value",
"datatype": "double",
},
{
"name": "quality",
"datatype": "integer"
}];
const boundlessTable = await infinityDatabase.createTable(TableDefinition);
const tableId = boundlessTable.TableIdentifier;
console.log("Table Created: " + tableId);- Insert Data
let ctr = 200;
let rows = [];
while (ctr > 0) {
rows.push({ "time": ctr, "tagid": ctr, "value": ctr, "quality": ctr })
ctr--;
}
console.time("Insertion");
const result = await boundlessTable.insert(rows);
console.timeEnd("Insertion");- Search Data
const filter = {
"conditions": [
{
"name": "value",
"operator": "=",
"values": [
10
]
},
{
"name": "quality",
"operator": "=",
"values": [
1
]
}
],
"combine": "$1:raw OR $2:raw"
}
const results = await boundlessTable.search(undefined, undefined, filter);
console.log(results);- Load Existing Table
const boundlessTable = await infinityDatabase.loadTable(TypeId);- Authors love for Open Source.
- pg-promise.
- ioredis.
- redis-scripto.
- New ideas/techniques are welcomed.
- Raise a Pull Request.
0.0.1[Beta]
This project is contrubution to public domain and completely free for use, view LICENSE.md file for details.
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent