JSoup web parser - P4rse
/* Hey folks. I decided to share with you this java website parser I made to make my web developing easier. If you need to generate SQL queries dynamically based on webpage indexing, or output website's source code into textfile, or even choose certain node based on class / id and dump it's content into text file, you might find this tool useful. You will need JSoup library to run this and if you forge it into executable JAR file, then all text files are generated in directory of JAR. */
import org.jsoup.*;
import java.io.*;
import java.net.MalformedURLException;
import java.net.UnknownHostException;
import java.util.Random;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
public class P4rse
{
private static Document target;
private static boolean connected = false;
private static String[] command_collection = new String[]{"connect","disconnect","sql","author","srcout","nodetext","exit"};
public static void main(String args[]) throws IOException
{
System.out.println("P4rse - visit websites not...\n");
P4rse.cycle();
}
public static void printwrite(String name, String[] values) throws IOException
{
PrintWriter writer;
name = name.replace("www","").replace(".","").replace("http://","");
System.out.println("Generating file...");
try
{
writer = new PrintWriter(name+".txt", "UTF-8");
}
catch(FileNotFoundException e)
{
Random rn = new Random();
writer = new PrintWriter((10000 + rn.nextInt(100000 - 10000 + 1))+".txt", "UTF-8");
}
System.out.println("Writing data...");
for(int i=0;i<=values.length-1;i++)
{
if(values[i] != null)
writer.println(values[i]);
else
continue;
}
writer.close();
}
public static boolean isNumeric(String str)
{
for (char c : str.toCharArray())
{
if (!Character.isDigit(c)) return false;
}
return true;
}
static int getTrailingInteger(String str)
{
int positionOfLastDigit = getPositionOfLastDigit(str);
if (positionOfLastDigit == str.length())
{
// string does not end in digits
return -1;
}
return Integer.parseInt(str.substring(positionOfLastDigit));
}
static int getPositionOfLastDigit(String str)
{
int pos;
for (pos=str.length()-1; pos>=0; --pos)
{
char c = str.charAt(pos);
if (!Character.isDigit(c)) break;
}
return pos + 1;
}
private static boolean connect(String url) throws IOException
{
System.out.println("Connecting...");
try
{
try
{
target = Jsoup.connect(url).get();
}catch(UnknownHostException ex){System.out.println("No internet connection.");return false;}
}catch(IllegalArgumentException|MalformedURLException e){System.out.println("Wrong URL format.");return false;}
return true;
}
private static void cycle() throws IOException
{
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
outerloop:
while(true)
{
System.out.print("P4rse > ");
String command = reader.readLine();
switch(command)
{
case "":
continue;
case "exit":
System.out.println("P4rse - visit websites not...");
break outerloop;
case "connect":
if(!P4rse.connected)
{
System.out.print("Target: ");
String tgt = reader.readLine();
if(P4rse.connect(tgt))
{
System.out.println("Connected to ("+tgt+").");
P4rse.connected = true;
}
else
{
break;
}
}
else
{
System.out.println("You have to disconnect first.");
}
break;
case "srcout":
if(P4rse.connected)
{
printwrite(target.location()+" source",new String[]{target.html()});
System.out.println("OK.");
}
else
{
System.out.println("You need to connect to target first.");
}
break;
case "nodetext":
if(P4rse.connected)
{
System.out.print("Insert nodecode: ");
String nodecode = reader.readLine();
Elements node = target.select(nodecode);
printwrite(target.location()+" node text",new String[]{node.text()});
System.out.println("OK.");
}
else
{
System.out.println("You need to connect to target first.");
}
break;
case "sql":
if(P4rse.connected)
{
System.out.print("Choose type: ");
String type = reader.readLine();
switch(type)
{
case "insert":
String lastchar = target.location().substring(target.location().length()-1, target.location().length());
System.out.println("Possible url increment: "+isNumeric(lastchar)+".");
System.out.print("Table name: ");
String table = reader.readLine();
System.out.print("Records amount: ");
int records = Integer.parseInt(reader.readLine());
String[] nodecode_set = new String[10];
for(int i=0;i<=records-1;i++)
{
System.out.print("Record "+i+" nodecode: ");
nodecode_set[i] = reader.readLine();
}
if(isNumeric(lastchar))
{
int urlinc = getTrailingInteger(target.location());
System.out.print("Delimiter: "+urlinc+" to ");
int delimiter = Integer.parseInt(reader.readLine());
String location_model = target.location().replace(Integer.toString(urlinc),"");
System.out.println("Starting simulation model...");
Thread t1 = new Thread(new Runnable()
{
public void run()
{
System.out.println("Simulation model started.");
Document document_model;
String[] query_storage = new String[((urlinc)-delimiter)+1];
String query = "";
int inc = -1;
for(int i=urlinc;i>=delimiter;i--)
{
try
{
document_model = Jsoup.connect(location_model+i).get();
System.out.println("Processing dummy target...");
inc++;
for(int q=0;q<=records-1;q++)
{
String text = document_model.select(nodecode_set[q]).text();
query = query + "'" + text + "',";
}
query = query.replaceAll(",$","");
String query_final = "INSERT INTO "+table+" VALUES("+query+");";
query_storage[inc] = query_final;
query = "";
}catch(IOException e){continue;}
}
try
{
printwrite(target.location()+" sql set",query_storage);
}catch(IOException e){e.printStackTrace();}
System.out.println("OK.");
System.out.print("Press enter to continue...");
}});
t1.start();
}
else
{
String query = "";
for(int i=0;i<=records-1;i++)
{
query = query + "'" + target.select(nodecode_set[i]).text() + "',";
}
query = query.replaceAll(",$","");
String query_final = "INSERT INTO "+table+" VALUES("+query+");";
printwrite(target.location()+" sql set",new String[]{query_final});
System.out.println("OK.");
}
break;
}
}
else
{
System.out.println("You need to connect to target first.");
}
break;
case "disconnect":
if(P4rse.connected)
{
P4rse.connected = false;
P4rse.target = null;
System.out.println("Disconnected.");
}
else
{
System.out.println("You are not conneced yet.");
}
break;
case "author":
System.out.println("Created by Andrej Noskaj 06.07 2016.");
break;
case "commands":
for(int i=0;i<=command_collection.length-1;i++)
{
System.out.println(command_collection[i]);
}
break;
default:
System.out.println("No such command. Try to type 'commands' if you struggle.");
break;
}
}
}
}
Comments