#
#				*** ddg.py ***
#
#	This is a Python library for fetching search results from DuckDuckGo.
#	It gets search results directly from links.duckduckgo.com.
#
from urllib.request import urlopen, Request
import re, json

headers = {'User-Agent': 'ddg.py'}

def loadPage(url):
	#
	# WARNING: The following line of code is necessary to make this program
	# a user agent rather than a robot. The user decides when and if
	# they want to load more pages. You are strongly encouraged not
	# to remove or "comment out" the following line.
	#
	input("\n[???] ENTER to fetch web page, CTRL+C to cancel ")
	page = urlopen(Request(url, headers=headers)).read().decode("utf-8")
	return page

def getVQD(page):
	return (re.search(',vqd="[0-9]-[0-9]*"', page)[0].replace(',vqd="', "")
		.replace('"', ""))
def fetchAll(search):
	resultsList = []

	searchTerm = search.replace(" ", "+")
	print("[DDG] Search term is: " + searchTerm)

	# Get the VQD of this search from the first human readable page
	print("[DDG] Fetching first human readable page...")
	currentUrl = "https://duckduckgo.com/?q=" + searchTerm + "&ia=web"
	currentPage = loadPage(currentUrl)
	print("[DDG] Extracting VQD number...")
	VQD = getVQD(currentPage)
	print("[DDG] VQD number is: " + VQD)

	# Use the VQD to access the links subdomain
	print("[DDG] Getting JSON format SERP from links.duckduckgo.com...")
	currentUrl = ("https://links.duckduckgo.com/d.js?q=" + searchTerm
			+ "&s=0&vqd=" + VQD)

	resultsFromLastPage = ['']

	while True:
		currentPage = loadPage(currentUrl)

		# Extract the results in JSON format
		try:
			JSONresultsString = (re.search(r"load\(\'d\'\,.*}]\);", currentPage)
				[0].replace("load('d',", "")[0:-2])
		except:
			break
			# Add the current page of JSON results to the results list
			resultsFromCurrentPage = json.loads(JSONresultsString.replace("\t", ""))
			if resultsFromCurrentPage[0:-1] == resultsFromLastPage[0:-1]:
				print("[DDG] Current page identical to last, assuming end reached")
				break
			resultsFromLastPage = resultsFromCurrentPage
			resultsList += resultsFromCurrentPage[0:-1]
			print("[DDG] Got " + str(len(resultsFromCurrentPage[0:-1]))
+ " results from current page")

			# Move to next page
			print("[DDG] Moving to next page...")
			try:
				currentUrl = ("https://links.duckduckgo.com/"
					+ resultsFromCurrentPage[-1]['n'])
			except:
				print("[DDG] End of results")
				break

		return resultsList
	if __name__ == "__main__":
		# If running as main program, get search term from user and tell user how
		# to use the results object
		results = fetchAll(input("\n[???] Search term: "))
		print("""
	[***] To look at the results, browse the list called 'results' using the below
	[***] console. For example, try typing:
	results[0]['a']
[***] This will show you the description of the first result.""")