Commit 4be91a97 authored by Jerry Ajay's avatar Jerry Ajay

Prototype

parent 8b1a57c6
*.swp
/build/*
GENERAL:
Clone the project and just type 'ant'.
Change build.xml as required.
FOLDERS:
/data - Contains data generated by JoinsFileGenerator.
/lib - Contains all libs used in this project.
/logs - Necessary logs for further investigation.
/scripts - python scripts to generate plots.
/src - Java code to parse the SQLLite queries.
NOTE:
Original data for generating the files in the data directory is obtained from
PhoneLab SQLLite logs.
<project name="SQLProcessor" basedir="." default="main">
<property name="src.dir" value="src"/>
<property name="build.dir" value="build"/>
<property name="lib.dir" value="lib"/>
<property name="classes.dir" value="${build.dir}/classes"/>
<property name="jar.dir" value="${build.dir}/jar"/>
<property name="main-class" value="SQLAnalyser.JoinsFileProcessor"/>
<property name="test-class" value="SQLAnalyser.TestRunner"/>
<path id="classpath">
<fileset dir="${lib.dir}" includes="**/*.jar"/>
</path>
<target name="clean">
<delete dir="${build.dir}"/>
</target>
<target name="compile">
<mkdir dir="${classes.dir}"/>
<javac srcdir="${src.dir}" destdir="${classes.dir}" classpathref="classpath"/>
</target>
<target name="jar" depends="compile">
<mkdir dir="${jar.dir}"/>
<jar destfile="${jar.dir}/${ant.project.name}.jar" basedir="${classes.dir}">
<manifest>
<attribute name="Main-Class" value="${main-class}"/>
</manifest>
</jar>
</target>
<target name="run" depends="jar">
<java fork="true" classname="${main-class}">
<classpath>
<path refid="classpath"/>
<path location="${jar.dir}/${ant.project.name}.jar"/>
</classpath>
<arg value="${arg0}"/>
</java>
</target>
<target name="test" depends="clean,jar">
<java fork="true" classname="${test-class}">
<classpath>
<path refid="classpath"/>
<path location="${jar.dir}/${ant.project.name}.jar"/>
</classpath>
</java>
</target>
<target name="clean-build" depends="clean,jar"/>
<target name="main" depends="clean,run"/>
</project>
This diff is collapsed.
This diff is collapsed.
edition 16
reminders 4
conversation_labels 367
stickers 2
game_instances 71
contacts 2
threads 1986
interest 1
LISTS 4
tree_entity 56
DataSources 76
ac_people 2169
games 19
alert 171
SHOULDKEEPON 2
collaborators 8
items 37
notifications 1
note_label 1
owner_sync_requests 55
images 371
event_pending_ops 1
blob_node 69
invitations 106
reminder 1
request_pending_ops 1
data 116
Account 17
messages 1075
view_data 194
sharing 18
LISTITEMS 6
notification 5
AccountMetadata121 2
accounts 3053
PrimePlaylistTracks 4
Message 236
application_sessions 1
pdu 1477
stream_item_photos 134
sms 1477
all_photos 14
MUSIC 7
list_item_conflict 9
section 46
leaderboard_pending_scores 1
pack_types 79
search_sequence 3987
raw_contacts 39
view_contacts 171
DocumentContent121 8
notes 5
list_item 238
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
import re
import numpy as np
import matplotlib.pyplot as plt
entries = []
def process_entries():
tables = []
hits = []
p = re.compile(r'^(?P<table>\w+)\s*(?P<hits>.*?)$')
for entry in entries:
m = p.match(entry)
tables.append(m.group('table'))
hits.append(int(m.group('hits')))
print tables
print hits
#ax.plot(hits)
#ax.bar(np.arange(len(hits)), hits, align='center', width=1.0)
#ax.set(xticks=range(len(tables)), xticklabels=tables, rotation='60')
plt.bar(range(len(hits)), hits, align='center', width=0.7)
plt.xticks(range(len(tables)), tables, size='small')
plt.setp(plt.xticks()[1], rotation=90)
plt.grid()
plt.xlabel("Table Name")
plt.ylabel("Hits");
plt.title("Table-Name vs Hits")
plt.show()
def readfile(filename):
f = open(filename, 'r')
for line in f:
entries.append(line.rstrip())
if __name__ == "__main__":
readfile("/home/jerry/Desktop/SQLLite-experiment/java2/data/"\
"table-ref-count.txt")
process_entries()
package SQLAnalyser;
import java.io.*;
import java.util.*;
import java.util.regex.*;
public class JoinsFileGenerator {
ArrayList<String> joins = new ArrayList<String>();
private void writeJOINsToFile() {
try {
File file = new File("data/joins.dat");
BufferedWriter output = new BufferedWriter(new FileWriter(file));
for(String join:joins) {
output.write(join);
output.write("\n");
}
output.close();
} catch(IOException e) {
System.out.println("Unable to write to file data/joins.dat");
}
}
private void extractJOINs(String line) {
Pattern p = Pattern.compile("\"(.*?)\"");
Matcher m;
if(line.contains("JOIN")) {
m = p.matcher(line);
if(m.find()){
String temp = m.group(1).replaceAll("\\\\n"," ");
//System.out.println(temp);
joins.add(temp);
}
}
}
private void parseFile(File file) {
try {
BufferedReader br = new BufferedReader(new FileReader(file));
String line;
while((line=br.readLine()) != null) {
extractJOINs(line);
}
} catch(IOException e) {
System.out.println("Unable to open file"+file.getAbsolutePath());
}
}
private void parseDirectory(String dirName) {
File dir = new File(dirName);
File[] dirListing = dir.listFiles();
for (File file:dirListing) {
this.parseFile(file);
}
this.writeJOINsToFile();
}
public static void main(String[] args) {
JoinsFileGenerator main = new JoinsFileGenerator();
main.parseDirectory(args[0]); //Assuming arg0 is the dirName
//main.parseFile(new File("data/joins-experiment.dat"));
}
}
package SQLAnalyser;
import java.io.*;
import java.util.*;
import net.sf.jsqlparser.*;
import net.sf.jsqlparser.parser.CCJSqlParserManager;
import net.sf.jsqlparser.statement.*;
import net.sf.jsqlparser.statement.select.*;
import SQLAnalyser.visitor.*;
public class JoinsFileProcessor {
private String sql;
private CCJSqlParserManager pm = new CCJSqlParserManager();
private void parseSQL() {
try {
Statement stmt = pm.parse(new StringReader(sql));
FileWriter fw;
try {
File unparsable = new File("data/parsable.dat");
unparsable.createNewFile();
fw = new FileWriter("data/parsable.dat",true);
fw.write(sql);
fw.write("\n");
fw.close();
} catch(IOException ioe) {
System.err.println(ioe.getMessage());
}
} catch(JSQLParserException e) {
FileWriter fw;
try {
File unparsable = new File("data/unparsable.dat");
unparsable.createNewFile();
fw = new FileWriter("data/unparsable.dat",true);
fw.write(sql);
fw.write("\n");
fw.close();
} catch(IOException ioe) {
System.err.println(ioe.getMessage());
}
}
}
public void parseJoinFile(String joinsFileName) {
try {
BufferedReader br = new BufferedReader(new FileReader(joinsFileName));
while((sql=br.readLine()) != null)
this.parseSQL();
} catch(IOException e) {
e.printStackTrace();
}
}
public void parseParsableJOINs(String filename) {
MySelectVisitor mySelectVisitor = new MySelectVisitor();
try {
BufferedReader br = new BufferedReader(new FileReader(filename));
while((sql=br.readLine()) != null) {
Statement stmt = pm.parse(new StringReader(sql));
if(stmt instanceof Select) {
SelectBody selectBody = ((Select)stmt).getSelectBody();
System.out.println("");
System.out.println("=================================");
System.out.println(((Select)stmt).toString());
selectBody.accept(new MySelectVisitor());
}
}
} catch(IOException | JSQLParserException e) {
e.printStackTrace();
}
}
public static void main(String[] args){
JoinsFileProcessor joinsFileProcessor = new JoinsFileProcessor();
//joinsFileProcessor.parseJoinFile(args[0]);
joinsFileProcessor.parseParsableJOINs("data/parsable.dat");
MyFromItemVisitor myFromItemVisitor = new MyFromItemVisitor();
myFromItemVisitor.printTableRefCount();
}
}
package SQLAnalyser;
import org.junit.runner.*;
import org.junit.runner.notification.*;
public class TestRunner {
public static void main(String[] args) {
Result result = JUnitCore.runClasses(JoinsFileGenerator.class);
for(Failure failure: result.getFailures()) {
System.out.println(failure.toString());
}
System.out.println(result.wasSuccessful());
}
}
package SQLAnalyser.visitor;
import java.util.*;
import net.sf.jsqlparser.*;
import net.sf.jsqlparser.schema.*;
import net.sf.jsqlparser.expression.*;
import net.sf.jsqlparser.statement.select.*;
import net.sf.jsqlparser.expression.operators.arithmetic.*;
import net.sf.jsqlparser.expression.operators.conditional.*;
import net.sf.jsqlparser.expression.operators.relational.*;
public class MyExpressionVisitor implements ExpressionVisitor {
@Override
public void visit(Addition addition) {
System.out.println("In ADDITION:");
}
@Override
public void visit(AllComparisonExpression allComparisonExpression) {
System.out.println("IN ALLCOMPARISON EXPRESSION:");
}
@Override
public void visit(AndExpression andExpression) {
System.out.println("IN ANDEXPRESSION:");
}
@Override
public void visit(AnyComparisonExpression anyComparisonExpression) {
System.out.println("IN ANYCOMPARISON EXPRESSION:");
}
@Override
public void visit(BitwiseXor bitwiseXor) {
System.out.println("IN BITWISE XOR:");
}
@Override
public void visit(BitwiseOr bitwiseOr) {
System.out.println("IN BITWISE OR:");
}
@Override
public void visit(BitwiseAnd bitwiseAnd) {
System.out.println("IN BITWISE AND:");
}
@Override
public void visit(Between between) {
System.out.println("IN BETWEEN:");
}
@Override
public void visit(CaseExpression caseExpression) {
System.out.println("IN CASEEXPRESSION:");
}
@Override
public void visit(Column column) {
System.out.println("IN COLUMN:");
System.out.println("COLUMN NAME:"+column.getColumnName());
System.out.println("WHOLE COLUMN NAME:"+column.getWholeColumnName());
//Table table = column.getTable();
//if(!table.toString().equals("null")) table.accept(new MyFromItemVisitor());
}
@Override
public void visit(Concat concat) {
System.out.println("IN CONCAT:");
System.out.println(concat.toString());
}
@Override
public void visit(DateValue dateValue) {
System.out.println("IN DATEVALUE:");
}
@Override
public void visit(Division division) {
System.out.println("IN DIVISION:");
}
@Override
public void visit(DoubleValue doubleValue) {
System.out.println("IN DOUBLEVALUE:");
}
@Override
public void visit(EqualsTo equalsTo) {
System.out.println("IN EQUALSTO:");
}
@Override
public void visit(ExistsExpression existsExpression) {
System.out.println("IN EXISTSEXPRESSION:");
}
@Override
public void visit(Function function) {
System.out.println("IN FUNCTION:");
}
@Override
public void visit(GreaterThan greaterThan) {
System.out.println("IN GREATER THAN:");
}
@Override
public void visit(GreaterThanEquals greaterThanEquals) {
System.out.println("IN GREATER THAN EQUALS:");
}
@Override
public void visit(InExpression inExpression) {
System.out.println("IN INEXPRESSION:");
}
@Override
public void visit(InverseExpression inverseExpression) {
System.out.println("IN INVERSE EXPRESSION:");
}
@Override
public void visit(IsNullExpression isNullExpression) {
System.out.println("IN ISNULL EXPRESSSION:");
}
@Override
public void visit(JdbcParameter jdbcParameter) {
System.out.println("IN JDBC PARAMETER:");
}
@Override
public void visit(LikeExpression likeExpression) {
System.out.println("IN LIKE EXPRESSION:");
}
@Override
public void visit(LongValue longValue) {
System.out.println("IN LONG VALUE");
}
@Override
public void visit(Matches matches) {
System.out.println("IN MATCHES:");
}
@Override
public void visit(MinorThan minorThan) {
System.out.println("IN MINOR THAN");
}
@Override
public void visit(MinorThanEquals minorThanEquals) {
System.out.println("IN MINOR THAN EQUALS:");
}
@Override
public void visit(Multiplication multiplication) {
System.out.println("IN MULTIPLICATION:");
}
@Override
public void visit(NotEqualsTo notEqualsTo) {
System.out.println("IN NOT EQUALS TO:");
}
@Override
public void visit(NullValue nullValue) {
System.out.println("IN NULL VALUE:");
}
@Override
public void visit(OrExpression orExpression) {
System.out.println("IN OR EXPRESSION:");
}
@Override
public void visit(Parenthesis parenthesis) {
System.out.println("IN PARENTHESIS:");
}
@Override
public void visit(StringValue stringValue) {
System.out.println("IN STRING VALUE:");
}
@Override
public void visit(SubSelect subSelect) {
System.out.println("IN SUBSELECT:");
System.out.println(subSelect.toString());
SelectBody selectBody = subSelect.getSelectBody();
selectBody.accept(new MySelectVisitor());
}
@Override
public void visit(Subtraction substraction) {
System.out.println("IN SUBTRACTION:");
}
@Override
public void visit(TimestampValue timestampValue) {
System.out.println("IN TIMESTAMP VALUE:");
}
@Override
public void visit(TimeValue timeValue) {
System.out.println("IN TIMEVALUE:");
}
@Override
public void visit(WhenClause whenClause) {
System.out.println("IN WHEN CLAUSE:");
}
}
package SQLAnalyser.visitor;
import java.util.*;
import net.sf.jsqlparser.statement.select.*;
import net.sf.jsqlparser.schema.*;
public class MyFromItemVisitor implements FromItemVisitor {
/*
* HashMap to keep count of the number of accesses to a table.
*/
static HashMap<String,Integer> tableRefCount =
new HashMap<String,Integer>(0);
@Override
public void visit(SubJoin subjoin) {
System.out.println("IN SUBJOIN:");
}
@Override
public void visit(SubSelect subSelect) {
System.out.println("IN SUBSELECT:");
System.out.println(subSelect.toString());
SelectBody selectBody = subSelect.getSelectBody();
selectBody.accept(new MySelectVisitor());
}
@Override
public void visit(Table tableName) {
System.out.println("IN TABLE:");
System.out.println(tableName.toString());
System.out.println("WholeTableName: "+tableName.getWholeTableName());
String alias = tableName.getAlias();
if(alias != null) System.out.println("Alias: "+alias);
String name = tableName.getName();
System.out.println("Name: "+name);
String schemaName = tableName.getSchemaName();
if(schemaName != null) System.out.println("SchemaName: "+schemaName);
/*
* Logic to update tableRefCount
*/
boolean found = false;
Integer count = 0;
for(String key:tableRefCount.keySet()){
if(key.equals(name)){
found=true;
count = tableRefCount.get(name);
count += 1;
}
}
if(!found) tableRefCount.put(name,1);
else tableRefCount.put(name, count);
//End logic
}
public void printTableRefCount() {
System.out.println("");
System.out.println("===================================================");
System.out.printf("%10sTABLE REFERENCE COUNT:\n"," ");
System.out.printf("TABLENAME%21sHITS\n"," ");