musikk / plyj Goto Github PK
View Code? Open in Web Editor NEWA Java parser written in Python using PLY.
License: Other
A Java parser written in Python using PLY.
License: Other
Dump the structure of a source file and get back to this guy.
We are attempting to parse Android source code and frequently encounter an error where the "tree" type is None. Below is the code and one of the many files which apparently can't be parsed, while numerous others have no problems. Any ideas?
The code in our project, in this case, couldn't be simpler:
j is a list element from a loop over all the Java files, ie: for j in java_files, same as all the working ones
comp_type is a string
import lib.plyj.parser as plyj
tree=''
parser = plyj.Parser()
def tree_parser(j,comp_type):
global tree
global parser
try:
tree=parser.parse_file(j) #This returns type(tree)==None on the file below, but works on numerous others
Any assistance you're willing to offer is appreciated.
package com.and;
import android.app.Activity;
import android.app.Dialog;
import android.content.Intent;
import android.os.Bundle;
import android.text.Editable;
import android.view.View;
import android.view.View.OnClickListener;
import android.widget.Button;
import android.widget.CheckBox;
import android.widget.EditText;
import android.widget.ImageView;
import android.widget.TextView;
import com.and.response.Loginresponse;
import com.and.service.Loginservice;
import com.and.util.FileUtil;
import com.and.util.Globals;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
public class MainActivity extends Activity
{
private CheckBox Fiddlercheckbox;
private CheckBox checkbox;
private EditText ip;
private Loginresponse loginresponse;
private Loginservice loginservice;
private Dialog myDialog;
private EditText password;
private ImageView settingImage;
private EditText username;
public void onCreate(Bundle paramBundle)
{
super.onCreate(paramBundle);
this.loginservice = new Loginservice();
if (!FileUtil.filesexistscheck("/data/data/com.and/files/Disclaimer.xml"))
{
this.myDialog = new Dialog(this);
this.myDialog.setContentView(2130903042);
this.myDialog.setTitle("Disclaimer");
((Button)this.myDialog.findViewById(2131034121)).setOnClickListener(new View.OnClickListener()
{
public void onClick(View paramView)
{
MainActivity.this.myDialog.dismiss();
try
{
OutputStreamWriter localOutputStreamWriter = new OutputStreamWriter(MainActivity.this.openFileOutput("Disclaimer.xml", 1));
localOutputStreamWriter.write("Yes");
localOutputStreamWriter.flush();
localOutputStreamWriter.close();
return;
}
catch (IOException localIOException)
{
localIOException.printStackTrace();
}
}
});
}
if ((FileUtil.filesexistscheck("/data/data/com.and/files/creds.xml")) && (!Globals.ServerIP.equals("")))
{
Intent localIntent = new Intent(this, loginactivity2.class);
localIntent.putExtra("check", "exists");
startActivity(localIntent);
finish();
}
setContentView(2130903045);
((Button)findViewById(2131034121)).setOnClickListener(new View.OnClickListener()
{
public void onClick(View paramView)
{
MainActivity.this.username = ((EditText)MainActivity.this.findViewById(2131034132));
MainActivity.this.password = ((EditText)MainActivity.this.findViewById(2131034117));
MainActivity.this.checkbox = ((CheckBox)MainActivity.this.findViewById(2131034134));
MainActivity.this.ip = ((EditText)MainActivity.this.findViewById(2131034133));
Globals.ServerIP = MainActivity.this.ip.getText().toString();
if ((Globals.ServerIP.equals("127.0.0.1")) || (Globals.ServerIP.equals("localhost")) || (Globals.ServerIP.equals("")))
{
System.out.print("ERROR: Invalid Server IP!");
MainActivity.this.myDialog = new Dialog(MainActivity.this);
MainActivity.this.myDialog.setContentView(2130903047);
MainActivity.this.myDialog.setTitle("Invalid Server IP!");
((TextView)MainActivity.this.myDialog.findViewById(2131034143)).setText("Remember to use the system's real IP rather than 127.0.0.1 or localhost.\n\n");
MainActivity.this.myDialog.setCancelable(true);
((Button)MainActivity.this.myDialog.findViewById(2131034113)).setOnClickListener(new View.OnClickListener()
{
public void onClick(View paramView)
{
MainActivity.this.myDialog.dismiss();
Intent localIntent = new Intent(MainActivity.this, MainActivity.class);
MainActivity.this.startActivity(localIntent);
MainActivity.this.finish();
}
});
MainActivity.this.myDialog.show();
return;
}
TextView localTextView;
if (MainActivity.this.checkbox.isChecked())
{
Globals.Httpsflag = Boolean.valueOf(true);
Globals.PostURL = "https://" + Globals.ServerIP + ":" + Globals.ServerPort + "/spring-ws-standalone/ws";
MainActivity.this.loginresponse = MainActivity.this.loginservice.send(MainActivity.this.username.getText().toString(), MainActivity.this.password.getText().toString());
if ((MainActivity.this.loginresponse != null) && (!MainActivity.this.loginresponse.getSessionid().equals("failed")))
break label573;
MainActivity.this.myDialog = new Dialog(MainActivity.this);
MainActivity.this.myDialog.setContentView(2130903047);
MainActivity.this.myDialog.setTitle("Login Failed!");
localTextView = (TextView)MainActivity.this.myDialog.findViewById(2131034143);
if ((MainActivity.this.loginresponse != null) || (!Globals.Httpsflag.booleanValue()))
break label518;
localTextView.setText("Could not connect to the server.\n\nCheck the user guide for Lessons 9 and 10. You'll have to hack some stuff first!");
}
while (true)
{
MainActivity.this.myDialog.setCancelable(true);
((Button)MainActivity.this.myDialog.findViewById(2131034113)).setOnClickListener(new View.OnClickListener()
{
public void onClick(View paramView)
{
MainActivity.this.myDialog.dismiss();
Intent localIntent = new Intent(MainActivity.this, MainActivity.class);
MainActivity.this.startActivity(localIntent);
MainActivity.this.finish();
}
});
MainActivity.this.myDialog.show();
return;
Globals.Httpsflag = Boolean.valueOf(false);
break;
label518: if (MainActivity.this.loginresponse == null)
{
localTextView.setText("Could not connect to the server.\n\n");
continue;
}
if (MainActivity.this.loginresponse.getSessionid().equals("failed"))
{
localTextView.setText("Please check the username and password then try again.\n\n");
continue;
}
localTextView.setText("Wierd Uknown Error!!! OH NOEZ!");
}
label573: Intent localIntent = new Intent(MainActivity.this, loginactivity2.class);
Globals.initialize(MainActivity.this.username.getText().toString(), MainActivity.this.loginresponse.getSessionid(), MainActivity.this.loginresponse.getAccountnumber());
localIntent.putExtra("Username", MainActivity.this.username.getText().toString());
localIntent.putExtra("check", "doesnot");
localIntent.putExtra("Password", MainActivity.this.password.getText().toString());
MainActivity.this.startActivity(localIntent);
MainActivity.this.finish();
}
});
this.settingImage = ((ImageView)findViewById(2131034135));
this.settingImage.setOnClickListener(new View.OnClickListener()
{
public void onClick(View paramView)
{
Intent localIntent = new Intent(MainActivity.this, Proxysetting.class);
MainActivity.this.startActivity(localIntent);
MainActivity.this.finish();
}
});
}
}
Is there a way to get the method name inside which a specific method I am looking for has been invoked? Other than that I want to be able to backtrace in general. For example, backtrace to a parameter definition which has been used in a method invocation or the method that calls the current method inside which I found a specific pattern or string of interest. Are there ways to do this?
Fix usage of name in methods p_primary_no_new_array3
and p_primary_no_new_array4
.
Hi.
I want to find out each method in a java file, then get the method's code.
I appreciate if you can help me with this.
Interface declare will cause this bug.
Traceback (most recent call last):
File "testVisitor.py", line 74, in <module>
tree.accept(MyVisitor())
File "/Users/sim/github/TAG_obfuscate/model.py", line 53, in accept
type_decl.accept(visitor)
File "/Users/sim/github/TAG_obfuscate/model.py", line 106, in accept
decl.accept(visitor)
File "/Users/sim/github/TAG_obfuscate/model.py", line 268, in accept
decl.accept(visitor)
File "/Users/sim/github/TAG_obfuscate/model.py", line 195, in accept
for e in self.body:
TypeError: 'NoneType' object is not iterable
In Python a class like so:
class Foo(object):
@staticmethod
def main(args):
pass
produces this AST:
Module(body=[
ClassDef(
name='Foo',
bases=[Name(id='object',ctx=Load())],
body=[
FunctionDef(
name='main',
args=arguments(
args=[Name(id='args',ctx=Param())],
vararg=None,
kwarg=None,
defaults=[]
),
body=[Pass()], # Look here
decorator_list=[Name(id='staticmethod',ctx=Load())])],
decorator_list=[]
)
])
A similar class in Java:
class Foo {
public static void main(String[] args) {
;
}
}
produces this AST with plyj:
CompilationUnit(
package_declaration=None,
import_declarations=[],
type_declarations=[
ClassDeclaration(
name='Foo',
body=[
MethodDeclaration(
name='main',
modifiers=['public', 'static'],
type_parameters=[],
parameters=[
FormalParameter(
variable=Variable(name='args', dimensions=0),
type=Type(
name=Name(value='String'),
type_arguments=[],
enclosed_in=None,
dimensions=1),
modifiers=[],
vararg=False)
],
return_type='void',
body=[None], # Look here
abrepract=False,
extended_dims=0,
throws=None)
],
modifiers=[],
type_parameters=[],
extends=None,
implements=[]
)
]
)
In the lines with the # Look here
comments, you can see a small but important difference. The Python AST has an ast node Pass
, but the plyj AST has a None
. It would be nicer if the None
could be replaced by something equivalent to Pass
as then we could add a visitor similar to ast.NodeVisitor
like:
class JavaNodeVisitor(object):
"""Visit nodes in the plyj.model.SourceElement
"""
# allow to be customizable
errormsg = "Expected a Java node or list, got '{0}'"
def __init__(self):
super(JavaNodeVisitor, self).__init__()
def generic_visit(self, node):
"""Called if no explicit visitor function exists for a node."""
for field, value in vars(node).iteritems():
if isinstance(value, list):
for item in value:
if isinstance(item, plyj.model.SourceElement):
self.visit(item)
elif isinstance(value, plyj.model.SourceElement):
self.visit(value)
def visit(self, node):
"""Visit a node."""
if not isinstance(node, plyj.model.SourceElement):
if isinstance(node, list):
return map(self.visit, node)
else:
# if we have a None the visitor fails here!
raise Exception(self.errormsg.format(classname(node)))
method = 'visit_' + classname(node)
visitor = getattr(self, method, self.generic_visit)
return visitor(node)
Also, being consistent is very important. So, if the None
is replaced by a Pass
object that inherits from plyj.model.SourceElement
, the consistency is better.
Got this traceback:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python2.7/dist-packages/plyj-0.0.1-py2.7.egg/plyj/parser.py", line 2036, in parse_file
return self.parse_string(content, debug=debug)
File "/usr/local/lib/python2.7/dist-packages/plyj-0.0.1-py2.7.egg/plyj/parser.py", line 2028, in parse_string
return self.parser.parse(prefix + code, lexer=self.lexer, debug=debug)
File "/usr/local/lib/python2.7/dist-packages/ply-3.4-py2.7.egg/ply/yacc.py", line 265, in parse
return self.parseopt_notrack(input,lexer,debug,tracking,tokenfunc)
File "/usr/local/lib/python2.7/dist-packages/ply-3.4-py2.7.egg/ply/yacc.py", line 971, in parseopt_notrack
p.callable(pslice)
File "/usr/local/lib/python2.7/dist-packages/plyj-0.0.1-py2.7.egg/plyj/parser.py", line 415, in p_primary_no_new_array4
p[0] = p[1] + '.' + p[3]
TypeError: unsupported operand type(s) for +: 'Name' and 'str'
Trying to parse this code (don't judge me; it's decompiled!):
import com.bulletphysics.collision.dispatch.CollisionObject;
public class zZ {
public long a;
public long b;
public CollisionObject a;
public CollisionObject b;
// $FF: synthetic field
private static boolean a = !zZ.class.desiredAssertionStatus();
public zZ(CollisionObject var1, CollisionObject var2, long var3, long var5) {
this.a = var1;
this.b = var2;
if(!a && var1 == var2) {
throw new AssertionError();
} else {
this.a = var3;
this.b = var5;
}
}
public boolean equals(Object var1) {
zZ var2;
return (var2 = (zZ)var1).a.equals(this.a) && var2.b.equals(this.b) || var2.b.equals(this.a) && var2.a.equals(this.b);
}
}
Support lambda expressions from java 8. The grammer can be snatched from antlr's java 8 grammar file: https://github.com/antlr/grammars-v4/blob/master/java8/Java8.g4
package com.example.java8;
import java.util.function.Supplier;
public class Main {
public static void main(String[] args) throws Exception {
Supplier<Integer> s = (() -> { return 42; });
}
}
fails with
>>> import plyj.parser as plyj
>>> parser = plyj.Parser()
>>> tree = parser.parse_file(file('path/to/Main.java'))
error: LexToken(),')',10,184)
As I refactored model.py for #10, to me it became obvious that there is a lack of unittests for the same.
As I looked into the code, I realized a lot of it is never tested (BAD)!
Here are the stats from coverage.py:
Name Stmts Miss Cover
---------------------------------
plyj\model 680 317 53%
plyj\parser 917 332 64%
---------------------------------
TOTAL 1597 649 59%
Obviously this is not good. In fact when you look at it more closely, you see that a good part of the 59% is the declarations of methods, classes etc.
That means that most of the "code" is un-executed during testing, from which we infer that if the implementation changes and breaks in future, the tests won't reveal it. This definitely has to be fixed.
Few things about testing: Read this and this. Try this tutorial too.
Are there any plans to add pretty printing?
overriding visit_MethodInvocation in Visitor class will not get all MethodInvocations. If a invoke is enclosed in ifelsethen or other block, it wouldn't accepte by the visit method.
Is this a bug? or just as the design.
plyj only supports Java 7 right now. Add support for all Java 8 features.
The branch java8_2
already contains default methods and an unsuccessful stab at reference expressions.
Hi there,
I'm using your parser for a project that I'm working on. Within that project, I have to parse large amounts Java code in order to find specific functions with certain characteristics (i.e. contain certain loops or recursive calls). Once I find those functions, I need to save them elsewhere and run them separately. To do this, though, I need to unparse the code from its parsed form so that it can be run as normal java code.
Do you have suggestions as to how I could take the parsed version of your code and unparse it back into java? Alternatively, is it possible to only partially parse, say, a ClassDeclaration, such that the "body" field is still in Java?
I recognize this may not be possible -- just trying to save myself a lot of work...
Thanks so much for making this! It's been super helpful so far.
I ran into an issue parsing files with dangling semicolons after an import statement. The java compilers I've tested seem to all accept these as valid syntax but plyj doesn't parse them correctly. I don't understand the codebase well enough to make the fix myself but the issue can be illustrated by modifying the import unit test in compilation_unit.py to something like this:
def test_import(self):
m = self.parser.parse_string('''
import foo;;
import foo.bar;
''')
self.assertEqual(m.import_declarations,
[model.ImportDeclaration(model.Name('foo')),
model.ImportDeclaration(model.Name('foo.bar'))])
Note the extra semicolon after import foo
. A 'real' example of this syntax can be found in AOSP's VideoDumpActivity.java
When using the MethodInvocation model, type arguments is always empty. Why does this field even exist? It would be super useful to have the type of each argument for differentiating between overloaded/polymorphic functions.
Populate type arguments list, or get rid of it.
This is a great tool for python. Actually, I wanna have the block comments in the java code.
Is it possible to that infomation?
Thanks,
WARNING: Token 'BLOCK_COMMENT' defined, but not used
WARNING: Token 'LINE_COMMENT' defined, but not used
WARNING: There are 2 unused tokens
Generating LALR tables
Could you please tell me in which format is your output generated?
Because when i a storing it in a .txt file,I am getting a line containing all the information.
Whereas what i want is something of this format:
CompilationUnit
TypeDeclaration
ClassDeclaration:(package private)
UnmodifiedClassDeclaration(Example)
ClassBody
ClassBodyDeclaration
MethodDeclaration:(package private)
ResultType
MethodDeclarator(bar)
FormalParameters
Block
BlockStatement
Statement
WhileStatement
Expression
PrimaryExpression
PrimaryPrefix
Name:baz
Statement
StatementExpression:null
PrimaryExpression
PrimaryPrefix
Name:buz.doSomething
PrimarySuffix
Arguments
Is it somehow possible?
I am trying to distribute some tool developed by using this module and now I need the rpm file to have this module installed in a distributed environment. Thanks!
I visited methodInvocation. But some method invocations are missed.
For example:
Missed:
public static final String LOG_TAG = ACRA.class.getSimpleName();
ErrorReporter localErrorReporter = ErrorReporter.getInstance();
Visited:
MethodInvocation(name='getApplicationContext', arguments=[], type_arguments=[], target=Name(value='paramReportsCrashes'))
MethodInvocation(name='d', arguments=[Name(value='LOG_TAG'), Additive(operator='+', lhs=Additive(operator='+', lhs=Literal(value='"ACRA is enabled for "'), rhs=MethodInvocation(name='getPackageName', arguments=[], type_arguments=[], target=Name(value='paramReportsCrashes'))), rhs=Literal(value='", intializing..."'))], type_arguments=[], target=Name(value='Log'))
MethodInvocation(name='init', arguments=[Name(value='paramReportsCrashes'), Name(value='paramBoolean')], type_arguments=[], target=Name(value='localErrorReporter'))
MethodInvocation(name='setDefaultUncaughtExceptionHandler', arguments=[Name(value='localErrorReporter')], type_arguments=[], target=Name(value='Thread'))
MethodInvocation(name='setReportSender', arguments=[InstanceCreation(type=Type(name=Name(value='HttpPostSender'), type_arguments=[], enclosed_in=None, dimensions=0), type_arguments=[], arguments=[Name(value='paramString')], body=[], enclosed_in=None)], type_arguments=[], target=Name(value='localErrorReporter'))
MethodInvocation(name='checkReportsOnApplicationStart', arguments=[], type_arguments=[], target=Name(value='localErrorReporter'))
package com.facebook.acra;
import android.content.Context;
import android.util.Log;
import com.facebook.acra.reporter.ReportsCrashes;
import com.facebook.acra.sender.HttpPostSender;
public class ACRA
{
public static final ReportField[] ALL_CRASH_REPORT_FIELDS = {ReportField.ATTACHMENT_ORIGINAL_SIZE };
public static final String LOG_TAG = ACRA.class.getSimpleName();
public static final String NULL_VALUE = "ACRA-NULL-STRING";
private static ReportsCrashes mReportsCrashes;
public ACRA() {}
public static ReportsCrashes getConfig()
{
return mReportsCrashes;
}
public static ErrorReporter init(ReportsCrashes paramReportsCrashes, String paramString, boolean paramBoolean)
{
ErrorReporter localErrorReporter = ErrorReporter.getInstance();
if (mReportsCrashes == null)
{
mReportsCrashes = paramReportsCrashes;
paramReportsCrashes = paramReportsCrashes.getApplicationContext();
Log.d(LOG_TAG, "ACRA is enabled for " + paramReportsCrashes.getPackageName() + ", intializing...");
localErrorReporter.init(paramReportsCrashes, paramBoolean);
Thread.setDefaultUncaughtExceptionHandler(localErrorReporter);
if (paramString != null) {
localErrorReporter.setReportSender(new HttpPostSender(paramString));
}
localErrorReporter.checkReportsOnApplicationStart();
}
return localErrorReporter;
}
}
Thanks for your help.
I have a java file with following sample code in it:
public class ClassName {
public void main(){
try {
} catch (SomeException e) {
addJavascriptInterface(1, 2);
}
}
}
Now I am implementing a MethodInvocation visitor by following the symbol_visitor.py example like this:
def visit_MethodInvocation(self, method_invocation):
dosomething
So for all the other cases except for the try and catch statements I am able to get the results.
I tried to figure out the problem and I suspected that Try Class itself has an accept definition. When I removed it, it just worked fine.
I'm trying to parse java files for a list of all the classes used. At the minute the parser seems to be missing the body of the Anonymous class so I can't take a look inside at the classes which are used.
This is an example of the part of the java code which is being missed:
private ServiceConnection mConnection = new ServiceConnection() {
public void onServiceConnected(ComponentName className, IBinder binder) {
Log.i("MockTest", "Service connected");
service = ((StringService.StringBinder) binder).getService();
messageTextView.setText(service.getString());
}
public void onServiceDisconnected(ComponentName className) {
Log.i("MockTest", "Service disconnected");
service = null;
}
};
Is this a feature you would be willing to add? I'm not normally a python programmer so I've not looking into coding this myself, but I could look into it if you don't have much time :-)
It's a cool project, and being able to pip install
it would be great.
I've found a line of code from which raises an exception when parsed from the Java source code java/com/sun/corba/se/impl/activation/ServerMain.java on line 151
Class argTypes[] = new Class[] { String[].class } ;
And the error is...
line 420, in p_primary_no_new_array4
p[0] = p[1] + '[' + p[2] + '].' + p[4]
TypeError: unsupported operand type(s) for +: 'Name' and 'str'
The feature can make a great use case to compose the parsed classes. So suppose if I wanted to know the exact argument in string form passed to a function I would first parse the java code get the argument and then compose it to java code to get the argument.
In example / symbols.py you produce a tree:
tree = p.parse_file(sys.argv[1])
print('declared types:')
for type_decl in tree.type_declarations:
I am having some trouble understanding this tree structure...It's not an abstract syntax tree and there are no uniform node types...would you mind elaborating? Thanks :-)
Excuse me, I want to track the line number of the code in the grammar analysis(yacc), but modifying your code according to the official document can not find the current number information in the result. What should I do?
Thank you very much for your help!!!!!!
def p_expression(p):
'expression : expression PLUS expression'
p.lineno(1) # Line number of the left expression
p.lineno(2) # line number of the PLUS operator
p.lineno(3) # line number of the right expression
...
start,end = p.linespan(3) # Start,end lines of the right expression
starti,endi = p.lexspan(3) # Start,end positions of right expression
Here is a sample file
/* IndexUF.java */
public class IndexUF<T extends Object> {
private Indexed<T>[] parent;
static private class Indexed<T> {
public int index;
public T value;
}
}
When I check the parse output of parent
's type's dimension, it is set too 0
so example below will fail the assertion.
index_uf = parser.parse_file(file('IndexUF.java'))
assert index_uf.type_declarations[0].body[0].type.dimensions == 1
I found this while testing the output of my pretty printer, and it turned out the issue was in the AST.
A declarative, efficient, and flexible JavaScript library for building user interfaces.
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
An Open Source Machine Learning Framework for Everyone
The Web framework for perfectionists with deadlines.
A PHP framework for web artisans
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
Some thing interesting about web. New door for the world.
A server is a program made to process requests and deliver data to clients.
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Some thing interesting about visualization, use data art
Some thing interesting about game, make everyone happy.
We are working to build community through open source technology. NB: members must have two-factor auth.
Open source projects and samples from Microsoft.
Google ❤️ Open Source for everyone.
Alibaba Open Source for everyone
Data-Driven Documents codes.
China tencent open source team.