var webPage=require('webpage');
var page=webPage.create();
var fs=require("fs");
phantom.onError = function(msg, trace) {
var msgStack = ['PHANTOM ERROR: ' + msg];
if (trace && trace.length)
{
msgStack.push('TRACE:');
trace.forEach(function(t)
{
msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line + (t.function ? ' (in function ' + t.function +')' : ''));
});
}
console.error(msgStack.join('\n'));
phantom.exit(1);
};
//Disable web security
page.settings.webSecurityEnabled = false;
page.settings.XSSAuditingEnabled = false;
page.onConsoleMessage=function(msg){
console.log(msg);
};
page.open('http://boards.4chan.org/trash/catalog', function(status) {
console.log("evaluating site specific js");
var ret=page.evaluate(function() {
var getImg=function(x)
{
var v=document.querySelector("#thumb-"+x);
if(v)
{
console.log("converting image "+v.src);
var canvas=document.createElement("canvas");
canvas.width=v.width;
canvas.height=v.height;
var ctx=canvas.getContext("2d");
ctx.drawImage(v,0,0,v.width,v.height);
return canvas.toDataURL("image/png").split(",")[1];
}
}
for(var x in catalog.threads)
{
catalog.threads[x].teaser=escape(catalog.threads[x].teaser);
catalog.threads[x].img=getImg(x);
}
return { catalog:catalog.threads,sort:catalog.order.alt/*, img:img*/};
});
var entries=ret.sort.map(function(v,i,a)
{
ret.catalog[v].id=v;
return ret.catalog[v];
});
console.log("Rendering images. Got "+entries.length);
entries.forEach(function(v,i,a)
{
console.log("rendering "+v.id);
fs.write("img/"+v.id+".png",atob(v.img),"wb");
});
console.log("Processing text content");
var s=[];
entries.forEach(function(v,i,a)
{
s.push([
v.id,
v.date,
v.file,
v.lr.date,
v.i,
v.r,
v.teaser
].join("\t"));
});
fs.write("threads.txt", s.join("\r\n"), "w");
console.log("All done");
phantom.exit();
});
Comments
0 B
|👍
/👎