import urllib2, socket
import urllib, urllib2
import time, datetime
import threading, Queue
import re
import StringIO, gzip
import sys
debug = False
def bug(line):
if debug == True:
print "Debug:: " + line
def queueThread():
global proxyCount
ts = time.time()
dt = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d-%H-%M-%S')
print 'Saving...'
fout = open("scrape.txt", "w")
while not workerQueue.empty():
fout.write(workerQueue.get() + "\n")
proxyCount+=1
fout.close()
def proxylist():
print "Grabbing..."
primary_url = "http://proxy-list.org/english/index.php?p="
urls = []
for i in range(1, 11):
urls.append(primary_url + str(i))
for url in urls:
try:
bug("grabbing " + "'" + url + "'")
opener = urllib2.build_opener()
opener.addheaders = [('Host', 'www.proxylisty.com'),
('Connection', 'keep-alive'),
('Cache-Control', 'max-age=0'),
('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'),
('Upgrade-Insecure-Requests', '1'),
('User-agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'),
('Referer', 'https://www.google.co.za/'),
('Accept-Encoding','gzip, deflate, sdch'),
('Accept-Language','en-US,en;q=0.8')]
response = opener.open(url, timeout=10)
compressedFile = StringIO.StringIO()
compressedFile.write(response.read())
compressedFile.seek(0)
decompessedFile = gzip.GzipFile(fileobj=compressedFile, mode='rb')
html = decompessedFile.read()
templs = re.findall(r'<li class="proxy">([1-99999].*)?</li>', html)
for line in templs:
workerQueue.put(line)
bug("proxylist() " + line)
except Exception, e:
if e.message == " ":
bug(e.message)
bug("Failed to grab " + "'" + url + "'")
else:
bug("Failed to grab " + "'" + url + "'")
def usproxy():
templs = []
url = "http://www.us-proxy.org/"
try:
bug("grabbing " + "'" + url + "'")
opener = urllib2.build_opener()
opener.addheaders = [('Host', 'www.proxylisty.com'),
('Connection', 'keep-alive'),
('Cache-Control', 'max-age=0'),
('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'),
('Upgrade-Insecure-Requests', '1'),
('User-agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'),
('Referer', 'https://www.google.co.za/'),
('Accept-Encoding','gzip, deflate, sdch'),
('Accept-Language','en-US,en;q=0.8')]
response = opener.open(url, timeout=10)
html = response.read()
templs = re.findall(r'<tr><td>(.*?)</td><td>', html)
templs2 = re.findall(r'</td><td>[1-99999].*?</td><td>', html)
for i in range(len(templs)):
temp = templs[i] + ":" + templs2[i].replace('</td><td>', '')
workerQueue.put(temp)
bug("usproxy() " + templs[i] + ":" + templs2[i].replace('</td><td>', ''))
except Exception, e:
if e.message == " ":
bug(e.message)
bug("Failed to grab " + "'" + url + "'")
else:
bug("Failed to grab " + "'" + url + "'")
def freeproxylist():
url = "http://free-proxy-list.net/"
try:
bug("grabbing " + "'" + url + "'")
opener = urllib2.build_opener()
opener.addheaders = [('Host', 'www.proxylisty.com'),
('Connection', 'keep-alive'),
('Cache-Control', 'max-age=0'),
('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'),
('Upgrade-Insecure-Requests', '1'),
('User-agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'),
('Referer', 'https://www.google.co.za/'),
('Accept-Encoding','gzip, deflate, sdch'),
('Accept-Language','en-US,en;q=0.8')]
response = opener.open(url, timeout=10)
html = response.read()
templs = re.findall(r'<tr><td>(.*?)</td><td>', html)
templs2 = re.findall(r'</td><td>[1-99999].*?</td><td>', html)
for i in range(len(templs)):
workerQueue.put(templs[i] + ":" + templs2[i].replace('</td><td>', ''))
bug("freeproxylist() " + templs[i] + ":" + templs2[i].replace('</td><td>', ''))
except Exception, e:
if e.message == " ":
bug(e.message)
bug("Failed to grab " + "'" + url + "'")
else:
bug("Failed to grab " + "'" + url + "'")
def coolproxy():
primary_url = "http://www.cool-proxy.net/proxies/http_proxy_list/sort:score/direction:desc/page:"
urls = []
for i in range(1, 13):
urls.append(primary_url + str(i))
for url in urls:
bug("grabbing " + "'" + url + "'")
try:
opener = urllib2.build_opener()
opener.addheaders = [('Host', 'www.proxylisty.com'),
('Connection', 'keep-alive'),
('Cache-Control', 'max-age=0'),
('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'),
('Upgrade-Insecure-Requests', '1'),
('User-agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'),
('Referer', 'https://www.google.co.za/'),
('Accept-Encoding','gzip, deflate, sdch'),
('Accept-Language','en-US,en;q=0.8')]
response = opener.open(url, timeout=10)
compressedFile = StringIO.StringIO()
compressedFile.write(response.read())
compressedFile.seek(0)
decompessedFile = gzip.GzipFile(fileobj=compressedFile, mode='rb')
html = decompessedFile.read()
templs = re.findall(r'str_rot13(.*?)</script>', html)
templs2 = re.findall(r'<td>[1-99999].*?</td>', html)
for i in range(len(templs)):
temp = templs[i].replace('("', '')#remove front of string
temp = temp.replace('")))', '')#remove back of string
temp = temp.decode('rot13').decode('base64')#decode from rot13 then from base64
workerQueue.put(temp + templs2[i].replace('<td>', ':').replace('</td>', ''))
bug("coolproxy() " + temp + templs2[i].replace('<td>', ':').replace('</td>', ''))
except Exception, e:
if e.message == " ":
bug(e.message)
bug("Failed to grab " + "'" + url + "'")
else:
bug("Failed to grab " + "'" + url + "'")
def proxylisty():
primary_url = "http://www.proxylisty.com/ip-proxylist-"
urls = []
for i in range(1, 68):
urls.append(primary_url + str(i))
for url in urls:
try:
bug("grabbing " + "'" + url + "'")
opener = urllib2.build_opener()
opener.addheaders = [('Host', 'www.proxylisty.com'),
('Connection', 'keep-alive'),
('Cache-Control', 'max-age=0'),
('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'),
('Upgrade-Insecure-Requests', '1'),
('User-agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'),
('Referer', 'https://www.google.co.za/'),
('Accept-Encoding','gzip, deflate, sdch'),
('Accept-Language','en-US,en;q=0.8')]
response = opener.open(url, timeout=10)
compressedFile = StringIO.StringIO()
compressedFile.write(response.read())
compressedFile.seek(0)
decompessedFile = gzip.GzipFile(fileobj=compressedFile, mode='rb')
html = decompessedFile.read()
templs = re.findall(r'<tr>\n<td>(.*?)</td>', html)
templs2 = re.findall(r'com/port/(.*?)-ip-list', html)
for i in range(len(templs)):
workerQueue.put(templs[i] + ":" + templs2[i])
bug("proxylisty() " + templs[i] + ":" + templs2[i])
except Exception, e:
if e.message == " ":
bug(e.message)
bug("Failed to grab " + "'" + url + "'")
else:
bug("Failed to grab " + "'" + url + "'")
if __name__ == "__main__":
print "Starting Proxy Scraper...\n"
proxyCount = 0
workerQueue = Queue.Queue()
tQueueThread = threading.Thread(target=queueThread)
tQueueThread.deamon = True
tProxylist = threading.Thread(target=proxylist)
tProxylist.deamon = True
tUsproxy = threading.Thread(target=usproxy)
tUsproxy.deamon = True
tFreeproxylist = threading.Thread(target=freeproxylist)
tFreeproxylist.deamon = True
tCoolproxy = threading.Thread(target=coolproxy)
tCoolproxy.deamon = True
tProxylisty = threading.Thread(target=proxylisty)
tProxylisty.deamon = True
tProxylist.start()
time.sleep(.500)
tUsproxy.start()
time.sleep(.500)
tFreeproxylist.start()
time.sleep(.500)
tCoolproxy.start()
time.sleep(.500)
tProxylisty.start()
time.sleep(2)
print "\nPlease wait..."
tProxylist.join()
tUsproxy.join()
tFreeproxylist.join()
tCoolproxy.join()
tProxylisty.join()
if not workerQueue.empty():
tQueueThread.start()
tQueueThread.join()
print "Saved to file!\n"
print "Proxies found: " + str(proxyCount)
else:
print "Could not scrape any proxies!"
print "Checking..."
time.sleep(2)
text_file = open("Working.txt", "w")
text_file.write("")
text_file.close()
socket.setdefaulttimeout(.4)
with open('scrape.txt') as f:
proxyList = f.readlines()
# read the list of proxy IPs in proxyList
#proxyList = ['172.30.1.1:8080', '172.30.3.3:8080'] # there are two sample proxy ip
def is_bad_proxy(pip):
try:
proxy_handler = urllib2.ProxyHandler({'http': pip})
opener = urllib2.build_opener(proxy_handler)
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
urllib2.install_opener(opener)
req=urllib2.Request('http://www.google.com') # change the url address here
sock=urllib2.urlopen(req)
except urllib2.HTTPError, e:
#print 'Error code: ', e.code
return e.code
except Exception, detail:
#print "ERROR:", detail
return 1
return 0
for item in proxyList:
if is_bad_proxy(item):
None
else:
print item
text_file = open("Working.txt", "a+")
text_file.write(item)
text_file.close()
Comments