Comments (7)
Fury fury = Fury.builder().withLanguage(Fury.Language.JAVA)
.withReferenceTracking(true).build();
SomeClass obj = new SomeClass();
byte[] bytes = fury.serialize(obj);
Object newobj = fury.deserialize(bytes);
SomeClass object = new SomeClass();
{
Fury fury = Fury.builder().withLanguage(Fury.Language.JAVA)
.withReferenceTracking(true).build();
byte[] bytes = fury.serialize(object);
Object o = fury.deserialize(bytes);
}
{
ThreadSafeFury fury = Fury.builder().withLanguage(Fury.Language.JAVA)
.withReferenceTracking(true).buildThreadSafeFury();
byte[] bytes = fury.serialize(object);
Object o = fury.deserialize(bytes);
}
{
ThreadSafeFury fury = new ThreadSafeFury(() -> {
Fury fury = Fury.builder().withLanguage(Fury.Language.JAVA)
.withReferenceTracking(true).build();
fury.register(xxx.class);
return fury;
});
byte[] bytes = fury.serialize(object);
Object o = fury.deserialize(bytes);
}
from blog.
Foo foo = Foo.create();
Encoder<Foo> encoder = Encoders.bean(Foo.class);
int initSize = 512;
MemoryBuffer buffer = MemoryUtils.buffer(initSize);
int numRows = 128;
VectorSchemaRoot root = ArrowUtils.
createVectorSchemaRoot(encoder.schema());
ArrowWriter arrowWriter = new ArrowWriter(root);
try (ArrowStreamWriter writer = new ArrowStreamWriter(
root, null, new FuryOutputStream(buffer))) {
writer.start();
for (int i = 0; i < numRows; i++) {
arrowWriter.write(encoder.toRow(foo));
}
arrowWriter.finish();
writer.writeBatch();
writer.end();
}
Foo foo = Foo.create();
Encoder<Foo> encoder = Encoders.bean(Foo.class);
int initSize = 512;
MemoryBuffer buffer = MemoryUtils.buffer(initSize);
int numRows = 128;
buffer.writerIndex(0);
ArrowWriter arrowWriter = ArrowUtils.createArrowWriter(encoder.schema());
for (int i = 0; i < numRows; i++) {
BinaryRow row = encoder.toRow(foo);
arrowWriter.write(row);
}
ArrowRecordBatch recordBatch = arrowWriter.finishAsRecordBatch();
DataTypes.serializeSchema(encoder.schema(), buffer);
ArrowUtils.serializeRecordBatch(recordBatch, buffer);
arrowWriter.reset();
ArrowStreamWriter.writeEndOfStream(new WriteChannel(Channels.newChannel(
new FuryOutputStream(buffer))), new IpcOption());
def test_record_batch(record_batch_bytes):
buf = pa.py_buffer(record_batch_bytes)
reader = pa.ipc.open_stream(buf)
foo_schema_without_meta = pa.schema(
[pa.field(f.name, f.type, f.nullable)
for f in foo_schema])
assert reader.schema == foo_schema_without_meta
batches = [batch for batch in reader]
assert len(batches) == 1
batch = batches[0]
encoder = pyfury.encoder(foo_schema)
writer = pyfury.ArrowWriter(foo_schema)
num_rows = 128
for _ in range(num_rows):
foo = create_foo()
writer.write(encoder.to_row(foo))
record_batch = writer.finish()
assert batch == record_batch
from blog.
public class Foo {
public int f1;
public String f2;
public List<String> f3;
public Map<String, Integer> f4;
public Bar f45;
}
public class Bar {
public int f1;
public String f2;
}
Encoder<Foo> encoder = Encoders.encoder(Foo);
public byte[] write(Foo foo) {
return encoder.toRow(foo).toBytes();
}
public Foo read(byte[] bytes) {
BinaryRow newRow = new BinaryRow(encoder.schema());
newRow.pointTo(bytes, 0, bytes.size());
return encoder.fromRow(newRow);
}
from blog.
Java API
public class Foo {
public int f1;
public Map<String, Bar> f2;
public Long f3;
}
public class Bar {
public String f1;
public List<String> f2;
}
Encoder<Foo> encoder = Encoders
.encoder(Foo.class);
byte[] bytes = encoder.toBytes(foo))
Python API
@dataclass
class Foo:
f1: pa.int32()
f2: Dict[str, 'Bar']
f3: pa.int64()
@dataclass
class Bar:
f1: str
f2: List[str]
encoder = pyfury.encoder(Foo)
new_foo: Foo = encoder\
.from_bytes(bytes_from_java)
Proto
message Foo {
int32 f1 = 1;
map<string, Bar> f2 = 2;
int64 f3 = 3;
}
message Bar {
string f1 = 1;
repeated string f2 = 2;
}
public static byte[] serialize(Foo foo) {
Map<String, Generated.Bar> pbF2 =
foo.f2.entrySet().stream()
.collect(Collectors.toMap(
Map.Entry::getKey, e -> {
Bar bar = e.getValue();
return Generated.Bar.newBuilder()
.setF1(bar.f1).addAllF2(bar.f2)
.build();
}));
return Generated.Foo.newBuilder()
.setF1(foo.f1)
.putAllF2(pbF2)
.setF3(foo.f3)
.build()
.toByteArray();
}
public Foo deserialize(byte[] bytes) throws Exception {
Generated.Foo pbFoo = Generated.Foo.parseFrom(bytes);
Foo foo = new Foo();
foo.f1 = pbFoo.getF1();
foo.f2 = pbFoo.getF2Map().entrySet().stream().collect(
Collectors.toMap(Map.Entry::getKey, e -> {
Generated.Bar value = e.getValue();
Bar bar = new Bar();
bar.f1 = value.getF1();
bar.f2 = new ArrayList<>(value.getF2List());
return bar;
}));
foo.f3 = pbFoo.getF3();
return foo;
}
from blog.
Old Java API
// Normal task
RayObject<String> hello = Ray.call(MyClass::echo, "hello");
RayObject<String> res = Ray.call(
new PyRemoteFunction<>("module", "func", String.class));
// Actor
ActorCreationOptions options = new ActorCreationOptions.Builder()
.setMaxRestarts(-1).createActorCreationOptions();
RayActor<MyActor> myActor = Ray.createActor(MyActor::new, 10, options);
Integer result = Ray.call(MyActor::increaseAndGet, actor, 1).get();
RayPyActor actor = Ray.createActor(new PyActorClass("module", "func"));
RayObject<byte[]> res = Ray.call(new PyActorMethod<>("fetch", byte[].class), actor);
New Java API
// Normal task
ObjectRef<String> obj = Ray.task(MyClass::echo, "hello").remote();
ObjectRef<String> res = Ray.task(
PyFunction.of("module", "func", String.class)).remote();
// Actor
ActorHandle<MyActor> myActor = Ray.actor(MyActor::new, 10)
.setMaxRestarts(-1).remote();
Integer obj = myActor.task(MyActor::increaseAndGet, 1).remote().get();
PyActorHandle pyActor = Ray.actor(PyActorClass.of("module", "Actor")).remote();
ObjectRef<byte[]> res = actor.task(PyActorMethod.of("fetch", byte[].class),
from blog.
Ray Java Call Python
public Foo foo(Bar bar, Map<String, List<Integer>> map){
return new Foo(bar, map);
}
Ray.task(PyFunction.of("ray_demo", "foo", Foo.class),
bar, map).remote();
pyActorHandle.task(...).remote(bar, map);
Ray Python Call Java
def foo(bar: Bar, map: Dict[str, List[pa.int32()]]) -> Foo:
return Foo(bar, map)
add_function = ray.java_function(
"io.ray.Demo", "foo")
foo_ref = add_function.remote(bar, map)
java_actor_handle.foo.remote(bar, map)
from blog.
Java API
public class Vertex {
public int index;
public Map<String, Double> props;
public Vertex to
}
fury.registerClass(Vertex.class, "example.Vertex");
List<Vertex> vertices = Arrays.asList(create(), create());
vertices.get(0).to = vertices.get(1);
vertices.get(1).to = vertices.get(0);
byte[] bytes = fury.serialize(vertices));
Python API
@dataclass
class Vertex:
index: pa.int32()
props: Dict[str, pa.float64()]
to: Vertex
fury.registerclass(Vertex, "example.Vertex")
data = [newvertex(), newvertex()]
data[0].to, data[1].to = data[1], data[0]
serialized = fury.serialize(data)
Golang API
type Vertex struct {
Index int32
Props map[string][float64]
To Foo
}
fury.RegisterType(*Vertex(nil), "example.Vertex")
data := []*Vertex{CreateVertex(), CreateVertex()}
data[0].to, data[1].to = data[1], data[0]
serialized := fury.serialize(data)
Decode
public class Vertex {
public int index;
public Map<String, Double> props;
public Vertex to
}
fury.registerClass(Vertex.class, "example.Vertex");
List<Vertex> vertices = fury.deserialize(binary));
Python API
@dataclass
class Vertex:
index: pa.int32()
props: Dict[str, pa.float64()]
to: Vertex
fury.registerclass(Vertex, "example.Vertex")
vertices = fury.deserialize(binary)
Golang API
type Vertex struct {
Index int32
Props map[string][float64]
To Foo
}
fury.RegisterType(*Vertex(nil), "example.Vertex")
vertices := fury.deserialize(binary).([]*Vertex)
from blog.
Related Issues (16)
- 《Ray: A Distributed Framework for Emerging AI Applications》论文阅读
- Photon: A Fast Query Engine for Lakehouse Systems
- Pandas Internals
- JVM Vectorization
- Tensor computation HOT 1
- Building alpa as an unified tensor engine
- GCC upgrade
- Apache Fury Report
- PySpark Notebook Setup Guide
- 《The Dataflow Model》论文阅读
- Apache Flink - Stateful Computations over Data Streams
- Apache Arrow
- Ray Streaming Cross-Language API Example HOT 1
- Deploy java jar to central repository
- Arrow Compute
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from blog.