Code Monkey home page Code Monkey logo

Comments (7)

chaokunyang avatar chaokunyang commented on June 19, 2024
Fury fury = Fury.builder().withLanguage(Fury.Language.JAVA)
         .withReferenceTracking(true).build();
SomeClass obj = new SomeClass();
byte[] bytes = fury.serialize(obj);
Object newobj = fury.deserialize(bytes);
SomeClass object = new SomeClass();
{
  Fury fury = Fury.builder().withLanguage(Fury.Language.JAVA)
      .withReferenceTracking(true).build();
  byte[] bytes = fury.serialize(object);
  Object o = fury.deserialize(bytes);
}
{
  ThreadSafeFury fury = Fury.builder().withLanguage(Fury.Language.JAVA)
      .withReferenceTracking(true).buildThreadSafeFury();
  byte[] bytes = fury.serialize(object);
  Object o = fury.deserialize(bytes);
}
{
  ThreadSafeFury fury = new ThreadSafeFury(() -> {
    Fury fury = Fury.builder().withLanguage(Fury.Language.JAVA)
        .withReferenceTracking(true).build();
    fury.register(xxx.class);
    return fury;
  });
  byte[] bytes = fury.serialize(object);
  Object o = fury.deserialize(bytes);
}

from blog.

chaokunyang avatar chaokunyang commented on June 19, 2024
Foo foo = Foo.create();
Encoder<Foo> encoder = Encoders.bean(Foo.class);
int initSize = 512;
MemoryBuffer buffer = MemoryUtils.buffer(initSize);
int numRows = 128;

VectorSchemaRoot root = ArrowUtils.
  createVectorSchemaRoot(encoder.schema());
ArrowWriter arrowWriter = new ArrowWriter(root);
try (ArrowStreamWriter writer = new ArrowStreamWriter(
    root, null, new FuryOutputStream(buffer))) {
  writer.start();
  for (int i = 0; i < numRows; i++) {
    arrowWriter.write(encoder.toRow(foo));
  }
  arrowWriter.finish();
  writer.writeBatch();
  writer.end();
}
Foo foo = Foo.create();
Encoder<Foo> encoder = Encoders.bean(Foo.class);
int initSize = 512;
MemoryBuffer buffer = MemoryUtils.buffer(initSize);
int numRows = 128;

buffer.writerIndex(0);
ArrowWriter arrowWriter = ArrowUtils.createArrowWriter(encoder.schema());
for (int i = 0; i < numRows; i++) {
  BinaryRow row = encoder.toRow(foo);
  arrowWriter.write(row);
}
ArrowRecordBatch recordBatch = arrowWriter.finishAsRecordBatch();
DataTypes.serializeSchema(encoder.schema(), buffer);
ArrowUtils.serializeRecordBatch(recordBatch, buffer);
arrowWriter.reset();
ArrowStreamWriter.writeEndOfStream(new WriteChannel(Channels.newChannel(
    new FuryOutputStream(buffer))), new IpcOption());
def test_record_batch(record_batch_bytes):
    buf = pa.py_buffer(record_batch_bytes)
    reader = pa.ipc.open_stream(buf)
    foo_schema_without_meta = pa.schema(
        [pa.field(f.name, f.type, f.nullable)
         for f in foo_schema])
    assert reader.schema == foo_schema_without_meta
    batches = [batch for batch in reader]
    assert len(batches) == 1
    batch = batches[0]

    encoder = pyfury.encoder(foo_schema)
    writer = pyfury.ArrowWriter(foo_schema)
    num_rows = 128
    for _ in range(num_rows):
        foo = create_foo()
        writer.write(encoder.to_row(foo))
    record_batch = writer.finish()
    assert batch == record_batch

from blog.

chaokunyang avatar chaokunyang commented on June 19, 2024
public class Foo {
  public int f1;
  public String f2;
  public List<String> f3;
  public Map<String, Integer> f4;
  public Bar f45;
}

public class Bar {
  public int f1;
  public String f2;
}

Encoder<Foo> encoder = Encoders.encoder(Foo);

public byte[] write(Foo foo) {
  return encoder.toRow(foo).toBytes();
}

public Foo read(byte[] bytes) {
  BinaryRow newRow = new BinaryRow(encoder.schema());
  newRow.pointTo(bytes, 0, bytes.size());
  return encoder.fromRow(newRow);
}

from blog.

chaokunyang avatar chaokunyang commented on June 19, 2024

Java API

public class Foo {
  public int f1;
  public Map<String, Bar> f2;
  public Long f3;
}

public class Bar {
  public String f1;
  public List<String> f2;
}

Encoder<Foo> encoder = Encoders
  .encoder(Foo.class);
byte[] bytes = encoder.toBytes(foo))

Python API

@dataclass
class Foo:
    f1: pa.int32()
    f2: Dict[str, 'Bar']
    f3: pa.int64()

@dataclass
class Bar:
    f1: str
    f2: List[str]

encoder = pyfury.encoder(Foo)
new_foo: Foo = encoder\
    .from_bytes(bytes_from_java)

Proto

message Foo {
    int32 f1 = 1;
    map<string, Bar> f2 = 2;
    int64 f3 = 3;
}

message Bar {
    string f1 = 1;
    repeated string f2 = 2;
} 
public static byte[] serialize(Foo foo) {
  Map<String, Generated.Bar> pbF2 =
      foo.f2.entrySet().stream()
      .collect(Collectors.toMap(
          Map.Entry::getKey, e -> {
        Bar bar = e.getValue();
        return Generated.Bar.newBuilder()
            .setF1(bar.f1).addAllF2(bar.f2)
            .build();
      }));
  return Generated.Foo.newBuilder()
      .setF1(foo.f1)
      .putAllF2(pbF2)
      .setF3(foo.f3)
      .build()
      .toByteArray();
}

public Foo deserialize(byte[] bytes) throws Exception {
  Generated.Foo pbFoo = Generated.Foo.parseFrom(bytes);
  Foo foo = new Foo();

  foo.f1 = pbFoo.getF1();
  foo.f2 = pbFoo.getF2Map().entrySet().stream().collect(
      Collectors.toMap(Map.Entry::getKey, e -> {
        Generated.Bar value = e.getValue();
        Bar bar = new Bar();
        bar.f1 = value.getF1();
        bar.f2 = new ArrayList<>(value.getF2List());
        return bar;
      }));
  foo.f3 = pbFoo.getF3();
  return foo;
}

from blog.

chaokunyang avatar chaokunyang commented on June 19, 2024

Old Java API

// Normal task
RayObject<String> hello = Ray.call(MyClass::echo, "hello");
RayObject<String> res = Ray.call(
        new PyRemoteFunction<>("module", "func", String.class));

// Actor
ActorCreationOptions options = new ActorCreationOptions.Builder()
    .setMaxRestarts(-1).createActorCreationOptions();
RayActor<MyActor> myActor = Ray.createActor(MyActor::new, 10, options);
Integer result = Ray.call(MyActor::increaseAndGet, actor, 1).get();
RayPyActor actor = Ray.createActor(new PyActorClass("module", "func"));
RayObject<byte[]> res = Ray.call(new PyActorMethod<>("fetch", byte[].class), actor);

New Java API

// Normal task
ObjectRef<String> obj = Ray.task(MyClass::echo, "hello").remote();
ObjectRef<String> res = Ray.task(
          PyFunction.of("module", "func", String.class)).remote();

// Actor
ActorHandle<MyActor> myActor = Ray.actor(MyActor::new, 10)
    .setMaxRestarts(-1).remote();
Integer obj = myActor.task(MyActor::increaseAndGet, 1).remote().get();
PyActorHandle pyActor = Ray.actor(PyActorClass.of("module", "Actor")).remote();
ObjectRef<byte[]> res = actor.task(PyActorMethod.of("fetch", byte[].class),

from blog.

chaokunyang avatar chaokunyang commented on June 19, 2024

Ray Java Call Python

public Foo foo(Bar bar, Map<String, List<Integer>> map){
  return new Foo(bar, map);
}

Ray.task(PyFunction.of("ray_demo", "foo", Foo.class),
  bar, map).remote();
pyActorHandle.task(...).remote(bar, map);

Ray Python Call Java

def foo(bar: Bar, map: Dict[str, List[pa.int32()]]) -> Foo:
    return Foo(bar, map)

add_function = ray.java_function(
    "io.ray.Demo", "foo")
foo_ref = add_function.remote(bar, map)
java_actor_handle.foo.remote(bar, map)

from blog.

chaokunyang avatar chaokunyang commented on June 19, 2024

Java API

public class Vertex {
  public int index;
  public Map<String, Double> props;
  public Vertex to
}
fury.registerClass(Vertex.class, "example.Vertex");
List<Vertex> vertices = Arrays.asList(create(), create());
vertices.get(0).to = vertices.get(1);
vertices.get(1).to = vertices.get(0);
byte[] bytes = fury.serialize(vertices));

Python API

@dataclass
class Vertex:
    index: pa.int32()
    props: Dict[str, pa.float64()]
    to: Vertex
fury.registerclass(Vertex, "example.Vertex")
data = [newvertex(), newvertex()]
data[0].to, data[1].to = data[1],  data[0]
serialized = fury.serialize(data)

Golang API

type Vertex struct {
  Index int32
  Props map[string][float64]       
  To Foo
}
fury.RegisterType(*Vertex(nil), "example.Vertex")
data := []*Vertex{CreateVertex(), CreateVertex()}
data[0].to, data[1].to = data[1],  data[0]
serialized := fury.serialize(data)

Decode

public class Vertex {
  public int index;
  public Map<String, Double> props;
  public Vertex to
}
fury.registerClass(Vertex.class, "example.Vertex");
List<Vertex> vertices = fury.deserialize(binary));

Python API

@dataclass
class Vertex:
    index: pa.int32()
    props: Dict[str, pa.float64()]
    to: Vertex
fury.registerclass(Vertex, "example.Vertex")
vertices = fury.deserialize(binary)

Golang API

type Vertex struct {
  Index int32
  Props map[string][float64]       
  To Foo
}
fury.RegisterType(*Vertex(nil), "example.Vertex")
vertices := fury.deserialize(binary).([]*Vertex)

from blog.

Related Issues (16)

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. 📊📈🎉

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google ❤️ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.