ddgr
Packaging status (expand)
● PyPI (pip3 install ddgr)
● Snap Store (snap install ddgr)
● Source Mage (cast ddgr)
● Termux (pip3 install ddgr)
which are not # closed. However, these tags play no structural role, and come # only in small quantities, so it's not really a problem. # # - All textual data (result title, result abstract, etc.) are # processed through a set of shared handlers. These handlers store # text in a shared buffer self.textbuf which can be retrieved and # cleared at appropriate times. # # Data (including charrefs and entityrefs) are ignored initially, # and when data needs to be recorded, the start_populating_textbuf # method is called to register the appropriate data, charref and # entityref handlers so that they append to self.textbuf. When # recording ends, pop_textbuf should be called to extract the text # and clear the buffer. stop_populating_textbuf returns the # handlers to their pristine state (ignoring data). # # Methods: # - start_populating_textbuf(self, data_transformer: Callable[[str], str]) -> None # - pop_textbuf(self) -> str # - stop_populating_textbuf(self) -> None # # - Outermost starttag and endtag handler methods: root_*. The whole # parser starts and ends in this state. # # - Each result is wrapped in a
tag with class "links_main".
#
#
# tag with class "result__title" contains the
# hyperlinked title.
#
#
#
#
#
# - For each result, the first tag with class "result__title" contains the
# hyperlinked title.
#
#
#
#
#
# - Abstracts are within the scope of tag with class "links_main". Links in
# abstract are ignored as they are available within tag.
#
#
#
#
#
# - Each title looks like
#
#
#
#
# result title
#
# file type (e.g. [PDF])
#
#
#
def __init__(self, offset=0):
html.parser.HTMLParser.__init__(self)
self.title = ''
self.url = ''
self.abstract = ''
self.filetype = ''
self.results = []
self.index = offset
self.textbuf = ''
self.click_result = ''
self.tag_annotations = {}
self.np_prev_button = ''
self.np_next_button = ''
self.npfound = False # First next params found
self.set_handlers_to('root')
self.vqd = '' # vqd returned from button, required for search query to get next set of results
# Tag handlers
@annotate_tag
def root_start(self, tag, attrs):
if tag == 'div':
if 'zci__result' in self.classes(attrs):
self.start_populating_textbuf()
return 'click_result'
if 'links_main' in self.classes(attrs):
# Initialize result field registers
self.title = ''
self.url = ''
self.abstract = ''
self.filetype = ''
self.set_handlers_to('result')
return 'result'
if 'nav-link' in self.classes(attrs):
self.set_handlers_to('input')
return 'input'
return ''
@retrieve_tag_annotation
def root_end(self, tag, annotation):
if annotation == 'click_result':
self.stop_populating_textbuf()
self.click_result = self.pop_textbuf()
self.set_handlers_to('root')
@annotate_tag
def result_start(self, tag, attrs):
if tag == 'h2' and 'result__title' in self.classes(attrs):
self.set_handlers_to('title')
return 'title'
if tag == 'a' and 'result__snippet' in self.classes(attrs) and 'href' in attrs:
self.start_populating_textbuf()
return 'abstract'
return ''
@retrieve_tag_annotation
def result_end(self, tag, annotation):
if annotation == 'abstract':
self.stop_populating_textbuf()
self.abstract = self.pop_textbuf()
elif annotation == 'result':
if self.url:
self.index += 1
result = Result(self.index, self.title, self.url, self.abstract, None)
self.results.append(result)
self.set_handlers_to('root')
@annotate_tag
def title_start(self, tag, attrs):
if tag == 'span':
# Print a space after the filetype indicator
self.start_populating_textbuf(lambda text: '[' + text + ']')
return 'title_filetype'
if tag == 'a' and 'href' in attrs:
# Skip 'News for', 'Images for' search links
if attrs['href'].startswith('/search'):
return ''
self.url = attrs['href']
try:
start = self.url.index('?q=') + len('?q=')
end = self.url.index('&sa=', start)
self.url = urllib.parse.unquote_plus(self.url[start:end])
except ValueError:
pass
self.start_populating_textbuf()
return 'title_link'
return ''
@retrieve_tag_annotation
def title_end(self, tag, annotation):
if annotation == 'title_filetype':
self.stop_populating_textbuf()
self.filetype = self.pop_textbuf()
self.start_populating_textbuf()
elif annotation == 'title_link':
self.stop_populating_textbuf()
self.title = self.pop_textbuf()
if self.filetype != '':
self.title = self.filetype + self.title
elif annotation == 'title':
self.set_handlers_to('result')
@annotate_tag
def abstract_start(self, tag, attrs):
if tag == 'span' and 'st' in self.classes(attrs):
self.start_populating_textbuf()
return 'abstract_text'
return ''
@retrieve_tag_annotation
def abstract_end(self, tag, annotation):
if annotation == 'abstract_text':
self.stop_populating_textbuf()
self.abstract = self.pop_textbuf()
elif annotation == 'abstract':
self.set_handlers_to('result')
@annotate_tag
def input_start(self, tag, attrs):
if tag == 'input' and 'name' in attrs:
if attrs['name'] == 'nextParams':
# The previous button always shows before next button
# If there's only 1 button (page 1), it's the next button
if self.npfound is True:
self.np_prev_button = self.np_next_button
else:
self.npfound = True
self.np_next_button = attrs['value']
return
if attrs['name'] == 'vqd' and attrs['value'] != '':
# vqd required to be passed for next/previous search page
self.vqd = attrs['value']
return
@retrieve_tag_annotation
def input_end(self, tag, annotation):
return
# Generic methods
# Set handle_starttag to SCOPE_start, and handle_endtag to SCOPE_end.
def set_handlers_to(self, scope):
self.handle_starttag = getattr(self, scope + '_start')
self.handle_endtag = getattr(self, scope + '_end')
def insert_annotation(self, tag, annotation):
if tag not in self.tag_annotations:
self.tag_annotations[tag] = []
self.tag_annotations[tag].append(annotation)
def start_populating_textbuf(self, data_transformer=None):
if data_transformer is None:
# Record data verbatim
self.handle_data = self.record_data
else:
def record_transformed_data(data):
self.textbuf += data_transformer(data)
self.handle_data = record_transformed_data
self.handle_entityref = self.record_entityref
self.handle_charref = self.record_charref
def pop_textbuf(self):
text = self.textbuf
self.textbuf = ''
return text
def stop_populating_textbuf(self):
self.handle_data = lambda data: None
self.handle_entityref = lambda ref: None
self.handle_charref = lambda ref: None
def record_data(self, data):
self.textbuf += data
def record_entityref(self, ref):
try:
self.textbuf += chr(html.entities.name2codepoint[ref])
except KeyError:
# Entity name not found; most likely rather sloppy HTML
# where a literal ampersand is not escaped; For instance,
# containing the following tag
#
#
expected market return s&p 500
#
# where &p is interpreted by HTMLParser as an entity (this
# behaviour seems to be specific to Python 2.7).
self.textbuf += '&' + ref
def record_charref(self, ref):
if ref.startswith('x'):
char = chr(int(ref[1:], 16))
else:
char = chr(int(ref))
self.textbuf += char
@staticmethod
def classes(attrs):
"""Get tag's classes from its attribute dict."""
return attrs.get('class', '').split()
def error(self, message):
raise NotImplementedError("subclasses of ParserBase must override error()")
Colors = collections.namedtuple('Colors', 'index, title, url, metadata, abstract, prompt, reset')
class Result:
"""
Container for one search result, with output helpers.
Parameters
----------
index : int or str
title : str
url : str
abstract : str
metadata : str, optional
Only applicable to DuckDuckGo News results, with publisher name and
publishing time.
Attributes
----------
index : str
title : str
url : str
abstract : str
metadata : str or None
Class Variables
---------------
colors : str
Methods
-------
print()
jsonizable_object()
urltable()
"""
# Class variables
colors = None
urlexpand = False
def __init__(self, index, title, url, abstract, metadata=None):
index = str(index)
self.index = index
self.title = title
self.url = url
self.abstract = abstract
self.metadata = metadata
self._urltable = {index: url}
def _print_title_and_url(self, index, title, url):
indent = INDENT - 2
colors = self.colors
if not self.urlexpand:
url = '[' + urllib.parse.urlparse(url).netloc + ']'
if colors:
# Adjust index to print result index clearly
print(" %s%-*s%s" % (colors.index, indent, index + '.', colors.reset), end='')
if not self.urlexpand:
print(' ' + colors.title + title + colors.reset + ' ' + colors.url + url + colors.reset)
else:
print(' ' + colors.title + title + colors.reset)
print(' ' * (INDENT) + colors.url + url + colors.reset)
else:
if self.urlexpand:
print(' %-*s %s' % (indent, index + '.', title))
print(' %s%s' % (' ' * (indent + 1), url))
else:
print(' %-*s %s %s' % (indent, index + '.', title, url))
def _print_metadata_and_abstract(self, abstract, metadata=None):
colors = self.colors
try:
columns, _ = os.get_terminal_size()
except OSError:
columns = 0
if metadata:
if colors:
print(' ' * INDENT + colors.metadata + metadata + colors.reset)
else:
print(' ' * INDENT + metadata)
if colors:
print(colors.abstract, end='')
if columns > INDENT + 1:
# Try to fill to columns
fillwidth = columns - INDENT - 1
for line in textwrap.wrap(abstract.replace('\n', ''), width=fillwidth):
print('%s%s' % (' ' * INDENT, line))
print('')
else:
print('%s\n' % abstract.replace('\n', ' '))
if colors:
print(colors.reset, end='')
def print(self):
"""Print the result entry."""
self._print_title_and_url(self.index, self.title, self.url)
self._print_metadata_and_abstract(self.abstract, metadata=self.metadata)
def print_paginated(self, display_index):
"""Print the result entry with custom index."""
self._print_title_and_url(display_index, self.title, self.url)
self._print_metadata_and_abstract(self.abstract, metadata=self.metadata)
def jsonizable_object(self):
"""Return a JSON-serializable dict representing the result entry."""
obj = {
'title': self.title,
'url': self.url,
'abstract': self.abstract
}
if self.metadata:
obj['metadata'] = self.metadata
return obj
def urltable(self):
"""Return a index-to-URL table for the current result.
Normally, the table contains only a single entry, but when the result
contains sitelinks, all sitelinks are included in this table.
Returns
-------
dict
A dict mapping indices (strs) to URLs (also strs).
"""
return self._urltable
class DdgCmdException(Exception):
pass
class NoKeywordsException(DdgCmdException):
pass
def require_keywords(method):
# Require keywords to be set before we run a DdgCmd method. If
# no keywords have been set, raise a NoKeywordsException.
@functools.wraps(method)
def enforced_method(self, *args, **kwargs):
if not self.keywords:
raise NoKeywordsException('No keywords.')
method(self, *args, **kwargs)
return enforced_method
def no_argument(method):
# Normalize a do_* method of DdgCmd that takes no argument to
# one that takes an arg, but issue a warning when an nonempty
# argument is given.
@functools.wraps(method)
def enforced_method(self, arg):
if arg:
method_name = arg.__name__
command_name = method_name[3:] if method_name.startswith('do_') else method_name
LOGGER.warning("Argument to the '%s' command ignored.", command_name)
method(self)
return enforced_method
class DdgCmd:
"""
Command line interpreter and executor class for ddgr.
Inspired by PSL cmd.Cmd.
Parameters
----------
opts : argparse.Namespace
Options and/or arguments.
Attributes
----------
options : argparse.Namespace
Options that are currently in effect. Read-only attribute.
keywords : str or list or strs
Current keywords. Read-only attribute
Methods
-------
fetch()
display_results(prelude='\n', json_output=False)
fetch_and_display(prelude='\n', json_output=False)
read_next_command()
help()
cmdloop()
"""
def __init__(self, opts, ua):
super().__init__()
self.cmd = ''
self.index = 0
self._opts = opts
self._ddg_url = DdgUrl(opts)
proxy = opts.proxy if hasattr(opts, 'proxy') else None
self._conn = DdgConnection(proxy=proxy, ua=ua)
self.results = []
self._urltable = {}
colors = self.colors
message = 'ddgr (? for help)'
self.prompt = ((colors.prompt + message + colors.reset + ' ')
if (colors and os.getenv('DISABLE_PROMPT_COLOR') is None) else (message + ': '))
@property
def options(self):
"""Current options."""
return self._opts
@property
def keywords(self):
"""Current keywords."""
return self._ddg_url.keywords
@require_keywords
def fetch(self, json_output=False):
"""Fetch a page and parse for results.
Results are stored in ``self.results``.
Parameters
----------
json_output : bool, optional
Whether to dump results in JSON format. Default is False.
Raises
------
DDGConnectionError
See Also
--------
fetch_and_display
"""
# This method also sets self._urltable.
page = self._conn.fetch_page(self._ddg_url)
if page is None:
return
if LOGGER.isEnabledFor(logging.DEBUG):
fd, tmpfile = tempfile.mkstemp(prefix='ddgr-response-')
os.close(fd)
with open(tmpfile, 'w', encoding='utf-8') as fp:
fp.write(page)
LOGDBG("Response body written to '%s'.", tmpfile)
if self._opts.num:
_index = len(self._urltable)
else:
_index = 0
self._urltable = {}
parser = DdgParser(offset=_index)
parser.feed(page)
if self._opts.num:
self.results.extend(parser.results)
else:
self.results = parser.results
for r in parser.results:
self._urltable.update(r.urltable())
self._ddg_url.np_prev = parser.np_prev_button
self._ddg_url.np_next = parser.np_next_button
self._ddg_url.vqd = parser.vqd
# Show instant answer
if self.index == 0 and parser.click_result and not json_output:
if self.colors:
print(self.colors.abstract)
try:
columns, _ = os.get_terminal_size()
except OSError:
columns = 0
fillwidth = columns - INDENT
for line in textwrap.wrap(parser.click_result.strip(), width=fillwidth):
print('%s%s' % (' ' * INDENT, line))
if self.colors:
print(self.colors.reset, end='')
LOGDBG('Prev nextParams: %s', self._ddg_url.np_prev)
LOGDBG('Next nextParams: %s', self._ddg_url.np_next)
LOGDBG('VQD: %s', self._ddg_url.vqd)
self._ddg_url.update_num(len(parser.results))
@require_keywords
def display_results(self, prelude='\n', json_output=False):
"""Display results stored in ``self.results``.
Parameters
----------
See `fetch_and_display`.
"""
if self._opts.num:
results = self.results[self.index:(self.index + self._opts.num)]
else:
results = self.results
if json_output:
# JSON output
results_object = [r.jsonizable_object() for r in results]
print(json.dumps(results_object, indent=2, sort_keys=True, ensure_ascii=False))
elif not results:
print('No results.', file=sys.stderr)
elif self._opts.num: # Paginated output
sys.stderr.write(prelude)
for i, r in enumerate(results):
r.print_paginated(str(i + 1))
else: # Regular output
sys.stderr.write(prelude)
for r in results:
r.print()
@require_keywords
def fetch_and_display(self, prelude='\n', json_output=False):
"""Fetch a page and display results.
Results are stored in ``self.results``.
Parameters
----------
prelude : str, optional
A string that is written to stderr before showing actual results,
usually serving as a separator. Default is an empty line.
json_output : bool, optional
Whether to dump results in JSON format. Default is False.
Raises
------
DDGConnectionError
See Also
--------
fetch
display_results
"""
self.fetch()
self.display_results(prelude=prelude, json_output=json_output)
def read_next_command(self):
"""Show omniprompt and read user command line.
Command line is always stripped, and each consecutive group of
whitespace is replaced with a single space character. If the
command line is empty after stripping, when ignore it and keep
reading. Exit with status 0 if we get EOF or an empty line
(pre-strip, that is, a raw ) twice in a row.
The new command line (non-empty) is stored in ``self.cmd``.
"""
enter_count = 0
while True:
try:
cmd = input(self.prompt)
except EOFError:
sys.exit(0)
if not cmd:
enter_count += 1
if enter_count == 2:
# Double
sys.exit(0)
else:
enter_count = 0
cmd = ' '.join(cmd.split())
if cmd:
self.cmd = cmd
break
@staticmethod
def help():
DdgArgumentParser.print_omniprompt_help(sys.stderr)
printerr('')
@require_keywords
@no_argument
def do_first(self):
if self._opts.num:
if self.index < self._opts.num:
print('Already at the first page.', file=sys.stderr)
else:
self.index = 0
self.display_results()
return
try:
self._ddg_url.first_page()
except ValueError as e:
print(e, file=sys.stderr)
return
self.fetch_and_display()
def do_ddg(self, arg):
if self._opts.num:
self.index = 0
self.results = []
self._urltable = {}
# Update keywords and reconstruct URL
self._opts.keywords = arg
self._ddg_url = DdgUrl(self._opts)
# If there is a Bang, let DuckDuckGo do the work
if arg[0] == '!' or (len(arg) > 1 and arg[1] == '!'):
open_url(self._ddg_url.full())
else:
self.fetch_and_display()
@require_keywords
@no_argument
def do_next(self):
if self._opts.num:
count = len(self.results)
if self._ddg_url._qrycnt == 0 and self.index >= count:
print('No results.', file=sys.stderr)
return
self.index += self._opts.num
if count - self.index < self._opts.num:
self._ddg_url.next_page()
self.fetch_and_display()
else:
self.display_results()
elif self._ddg_url._qrycnt == 0:
# If no results were fetched last time, we have hit the last page already
print('No results.', file=sys.stderr)
else:
self._ddg_url.next_page()
self.fetch_and_display()
def handle_range(self, nav, low, high):
try:
if self._opts.num:
vals = [int(x) + self.index for x in nav.split('-')]
else:
vals = [int(x) for x in nav.split('-')]
if len(vals) != 2:
printerr('Invalid range %s.' % nav)
return
if vals[0] > vals[1]:
vals[0], vals[1] = vals[1], vals[0]
for _id in range(vals[0], vals[1] + 1):
if self._opts.num and _id not in range(low, high):
printerr('Invalid index %s.' % (_id - self.index))
continue
if str(_id) in self._urltable:
open_url(self._urltable[str(_id)])
else:
printerr('Invalid index %s.' % _id)
except ValueError:
printerr('Invalid range %s.' % nav)
@require_keywords
def do_open(self, low, high, *args):
if not args:
printerr('Index or range missing.')
return
for nav in args:
if nav == 'a':
for key, _ in sorted(self._urltable.items()):
if self._opts.num and int(key) not in range(low, high):
continue
open_url(self._urltable[key])
elif nav in self._urltable:
if self._opts.num:
nav = str(int(nav) + self.index)
if int(nav) not in range(low, high):
printerr('Invalid index %s.' % (int(nav) - self.index))
continue
open_url(self._urltable[nav])
elif '-' in nav:
self.handle_range(nav, low, high)
else:
printerr('Invalid index %s.' % nav)
@require_keywords
@no_argument
def do_previous(self):
if self._opts.num:
if self.index < self._opts.num:
print('Already at the first page.', file=sys.stderr)
else:
self.index -= self._opts.num
self.display_results()
return
try:
self._ddg_url.prev_page()
except ValueError as e:
print(e, file=sys.stderr)
return
self.fetch_and_display()
def copy_url(self, idx):
try:
content = self._urltable[str(idx)].encode('utf-8')
# try copying the url to clipboard using native utilities
copier_params = []
if sys.platform.startswith(('linux', 'freebsd', 'openbsd')):
if shutil.which('xsel') is not None:
copier_params = ['xsel', '-b', '-i']
elif shutil.which('xclip') is not None:
copier_params = ['xclip', '-selection', 'clipboard']
elif shutil.which('wl-copy') is not None:
copier_params = ['wl-copy']
# If we're using Termux (Android) use its 'termux-api'
# add-on to set device clipboard.
elif shutil.which('termux-clipboard-set') is not None:
copier_params = ['termux-clipboard-set']
elif sys.platform == 'darwin':
copier_params = ['pbcopy']
elif sys.platform == 'win32':
copier_params = ['clip']
elif sys.platform.startswith('haiku'):
copier_params = ['clipboard', '-i']
if copier_params:
Popen(copier_params, stdin=PIPE, stdout=DEVNULL, stderr=DEVNULL).communicate(content)
return
# If native clipboard utilities are absent, try to use terminal multiplexers
# tmux
if os.getenv('TMUX_PANE'):
copier_params = ['tmux', 'set-buffer']
Popen(copier_params + [content], stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL).communicate()
print('URL copied to tmux buffer.')
return
# GNU Screen paste buffer
if os.getenv('STY'):
copier_params = ['screen', '-X', 'readbuf', '-e', 'utf8']
tmpfd, tmppath = tempfile.mkstemp()
try:
with os.fdopen(tmpfd, 'wb') as fp:
fp.write(content)
copier_params.append(tmppath)
Popen(copier_params, stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL).communicate()
finally:
os.unlink(tmppath)
return
printerr('failed to locate suitable clipboard utility')
except Exception:
raise NoKeywordsException
def cmdloop(self):
"""Run REPL."""
if self.keywords:
if self.keywords[0][0] == '!' or (
len(self.keywords[0]) > 1 and self.keywords[0][1] == '!'
):
open_url(self._ddg_url.full())
else:
self.fetch_and_display()
while True:
self.read_next_command()
# Automatic dispatcher
#
# We can't write a dispatcher for now because that could
# change behaviour of the prompt. However, we have already
# laid a lot of ground work for the dispatcher, e.g., the
# `no_argument' decorator.
_num = self._opts.num
try:
cmd = self.cmd
if cmd == 'f':
self.do_first('')
elif cmd.startswith('d '):
self.do_ddg(cmd[2:])
elif cmd == 'n':
self.do_next('')
elif cmd.startswith('o '):
self.do_open(self.index + 1, self.index + self._opts.num + 1, *cmd[2:].split())
elif cmd.startswith('O '):
open_url.override_text_browser = True
self.do_open(self.index + 1, self.index + self._opts.num + 1, *cmd[2:].split())
open_url.override_text_browser = False
elif cmd == 'p':
self.do_previous('')
elif cmd == 'q':
break
elif cmd == '?':
self.help()
elif _num and cmd.isdigit() and int(cmd) in range(1, _num + 1):
open_url(self._urltable[str(int(cmd) + self.index)])
elif _num == 0 and cmd in self._urltable:
open_url(self._urltable[cmd])
elif self.keywords and cmd.isdigit() and int(cmd) < 100:
printerr('Index out of bound. To search for the number, use d.')
elif cmd == 'x':
Result.urlexpand = not Result.urlexpand
self.display_results()
elif cmd.startswith('c ') and cmd[2:].isdigit():
idx = int(cmd[2:])
if 0 < idx <= min(self._opts.num, len(self._urltable)):
self.copy_url(int(self.index) + idx)
else:
printerr("invalid index")
else:
self.do_ddg(cmd)
except KeyError:
printerr('Index out of bound. To search for the number, use d.')
except NoKeywordsException:
printerr('Initiate a query first.')
class DdgArgumentParser(argparse.ArgumentParser):
"""Custom argument parser for ddgr."""
# Print omniprompt help
@staticmethod
def print_omniprompt_help(file=None):
file = sys.stderr if file is None else file
file.write(textwrap.dedent("""
omniprompt keys:
n, p, f fetch the next, prev or first set of search results
index open the result corresponding to index in browser
o [index|range|a ...] open space-separated result indices, ranges or all
O [index|range|a ...] like key 'o', but try to open in a GUI browser
d keywords new DDG search for 'keywords' with original options
should be used to search omniprompt keys and indices
x toggle url expansion
c index copy url to clipboard
q, ^D, double Enter exit ddgr
? show omniprompt help
* other inputs are considered as new search keywords
"""))
# Print information on ddgr
@staticmethod
def print_general_info(file=None):
file = sys.stderr if file is None else file
file.write(textwrap.dedent("""
Version %s
Copyright © 2016-2020 Arun Prakash Jana
License: GPLv3
Webpage: https://github.com/jarun/ddgr
""" % _VERSION_))
# Augment print_help to print more than synopsis and options
def print_help(self, file=None):
super().print_help(file)
self.print_omniprompt_help(file)
self.print_general_info(file)
# Automatically print full help text on error
def error(self, message):
sys.stderr.write('%s: error: %s\n\n' % (self.prog, message))
self.print_help(sys.stderr)
self.exit(2)
# Type guards
@staticmethod
def positive_int(arg):
"""Try to convert a string into a positive integer."""
try:
n = int(arg)
assert n > 0
return n
except (ValueError, AssertionError):
raise argparse.ArgumentTypeError('%s is not a positive integer' % arg)
@staticmethod
def nonnegative_int(arg):
"""Try to convert a string into a nonnegative integer <= 25."""
try:
n = int(arg)
assert n >= 0
assert n <= 25
return n
except (ValueError, AssertionError):
raise argparse.ArgumentTypeError('%s is not a non-negative integer <= 25' % arg)
@staticmethod
def is_duration(arg):
"""Check if a string is a valid duration accepted by DuckDuckGo.
A valid duration is of the form dNUM, where d is a single letter h
(hour), d (day), w (week), m (month), or y (year), and NUM is a
non-negative integer.
"""
try:
if arg[0] not in ('h', 'd', 'w', 'm', 'y') or int(arg[1:]) < 0:
raise ValueError
except (TypeError, IndexError, ValueError):
raise argparse.ArgumentTypeError('%s is not a valid duration' % arg)
return arg
@staticmethod
def is_colorstr(arg):
"""Check if a string is a valid color string."""
try:
assert len(arg) == 6
for c in arg:
assert c in COLORMAP
except AssertionError:
raise argparse.ArgumentTypeError('%s is not a valid color string' % arg)
return arg
# Miscellaneous functions
def python_version():
return '%d.%d.%d' % sys.version_info[:3]
def get_colorize(colorize):
if colorize == 'always':
return True
if colorize == 'auto':
return sys.stdout.isatty()
# colorize = 'never'
return False
def set_win_console_mode():
# VT100 control sequences are supported on Windows 10 Anniversary Update and later.
# https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
# https://docs.microsoft.com/en-us/windows/console/setconsolemode
if platform.release() == '10':
STD_OUTPUT_HANDLE = -11
STD_ERROR_HANDLE = -12
ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004
try:
from ctypes import windll, wintypes, byref
kernel32 = windll.kernel32
for nhandle in (STD_OUTPUT_HANDLE, STD_ERROR_HANDLE):
handle = kernel32.GetStdHandle(nhandle)
old_mode = wintypes.DWORD()
if not kernel32.GetConsoleMode(handle, byref(old_mode)):
raise RuntimeError('GetConsoleMode failed')
new_mode = old_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING
if not kernel32.SetConsoleMode(handle, new_mode):
raise RuntimeError('SetConsoleMode failed')
# Note: No need to restore at exit. SetConsoleMode seems to
# be limited to the calling process.
except Exception:
pass
# Query autocompleter
# This function is largely experimental and could raise any exception;
# you should be prepared to catch anything. When it works though, it
# returns a list of strings the prefix could autocomplete to (however,
# it is not guaranteed that they start with the specified prefix; for
# instance, they won't if the specified prefix ends in a punctuation
# mark.)
def completer_fetch_completions(prefix):
# One can pass the 'hl' query param to specify the language. We
# ignore that for now.
api_url = ('https://duckduckgo.com/ac/?q=%s&kl=wt-wt' %
urllib.parse.quote(prefix, safe=''))
# A timeout of 3 seconds seems to be overly generous already.
resp = urllib.request.urlopen(api_url, timeout=3)
respobj = json.loads(resp.read().decode('utf-8'))
return [entry['phrase'] for entry in respobj]
def completer_run(prefix):
if prefix:
completions = completer_fetch_completions('+'.join(prefix.split()))
if completions:
print('\n'.join(completions))
sys.exit(0)
def parse_args(args=None, namespace=None):
"""Parse ddgr arguments/options.
Parameters
----------
args : list, optional
Arguments to parse. Default is ``sys.argv``.
namespace : argparse.Namespace
Namespace to write to. Default is a new namespace.
Returns
-------
argparse.Namespace
Namespace with parsed arguments / options.
"""
colorstr_env = os.getenv('DDGR_COLORS')
argparser = DdgArgumentParser(description='DuckDuckGo from the terminal.')
addarg = argparser.add_argument
addarg('-n', '--num', type=argparser.nonnegative_int, default=10, metavar='N',
help='show N (0<=N<=25) results per page (default 10); N=0 shows actual number of results fetched per page')
addarg('-r', '--reg', dest='region', default='us-en', metavar='REG',
help="region-specific search e.g. 'us-en' for US (default); visit https://duckduckgo.com/params")
addarg('--colorize', nargs='?', choices=['auto', 'always', 'never'],
const='always', default='auto',
help="""whether to colorize output; defaults to 'auto', which enables
color when stdout is a tty device; using --colorize without an argument
is equivalent to --colorize=always""")
addarg('-C', '--nocolor', action='store_true', help='equivalent to --colorize=never')
addarg('--colors', dest='colorstr', type=argparser.is_colorstr, default=colorstr_env if colorstr_env else 'oCdgxy', metavar='COLORS',
help='set output colors (see man page for details)')
addarg('-j', '--ducky', action='store_true', help='open the first result in a web browser; implies --np')
addarg('-t', '--time', dest='duration', metavar='SPAN', default='', choices=('d', 'w', 'm', 'y'), help='time limit search '
'[d (1 day), w (1 wk), m (1 month), y (1 year)]')
addarg('-w', '--site', dest='sites', action='append', metavar='SITE', help='search sites using DuckDuckGo')
addarg('-x', '--expand', action='store_true', help='Show complete url in search results')
addarg('-p', '--proxy', metavar='URI', help='tunnel traffic through an HTTPS proxy; URI format: [http[s]://][user:pwd@]host[:port]')
addarg('--unsafe', action='store_true', help='disable safe search')
addarg('--noua', action='store_true', help='disable user agent')
addarg('--json', action='store_true', help='output in JSON format; implies --np')
addarg('--gb', '--gui-browser', dest='gui_browser', action='store_true', help='open a bang directly in gui browser')
addarg('--np', '--noprompt', dest='noninteractive', action='store_true', help='perform search and exit, do not prompt')
addarg('--url-handler', metavar='UTIL', help='custom script or cli utility to open results')
addarg('--show-browser-logs', action='store_true', help='do not suppress browser output (stdout and stderr)')
addarg('-v', '--version', action='version', version=_VERSION_)
addarg('-d', '--debug', action='store_true', help='enable debugging')
addarg('keywords', nargs='*', metavar='KEYWORD', help='search keywords')
addarg('--complete', help=argparse.SUPPRESS)
parsed = argparser.parse_args(args, namespace)
if parsed.nocolor:
parsed.colorize = 'never'
return parsed
def main():
opts = parse_args()
# Set logging level
if opts.debug:
LOGGER.setLevel(logging.DEBUG)
LOGDBG('ddgr version %s Python version %s', _VERSION_, python_version())
# Handle query completer
if opts.complete is not None:
completer_run(opts.complete)
check_stdout_encoding()
# Add cmdline args to readline history
if opts.keywords:
try:
readline.add_history(' '.join(opts.keywords))
except Exception:
pass
# Set colors
colorize = get_colorize(opts.colorize)
colors = Colors(*[COLORMAP[c] for c in opts.colorstr], reset=COLORMAP['x']) if colorize else None
Result.colors = colors
Result.urlexpand = opts.expand
DdgCmd.colors = colors
# Try to enable ANSI color support in cmd or PowerShell on Windows 10
if sys.platform == 'win32' and sys.stdout.isatty() and colorize:
set_win_console_mode()
if opts.url_handler is not None:
open_url.url_handler = opts.url_handler
else:
open_url.override_text_browser = bool(opts.gui_browser)
# Handle browser output suppression
open_url.suppress_browser_output = not (opts.show_browser_logs or (os.getenv('BROWSER') in TEXT_BROWSERS))
try:
repl = DdgCmd(opts, '' if opts.noua else USER_AGENT)
if opts.json or opts.ducky or opts.noninteractive:
# Non-interactive mode
if repl.keywords and (
repl.keywords[0][0] == '!' or
(len(repl.keywords[0]) > 1 and repl.keywords[0][1] == '!')
):
# Handle bangs
open_url(repl._ddg_url.full())
else:
repl.fetch(opts.json)
if opts.ducky:
if repl.results:
open_url(repl.results[0].url)
else:
print('No results.', file=sys.stderr)
else:
repl.display_results(prelude='', json_output=opts.json)
sys.exit(0)
# Interactive mode
repl.cmdloop()
except Exception as e:
# If debugging mode is enabled, let the exception through for a traceback;
# otherwise, only print the exception error message.
if LOGGER.isEnabledFor(logging.DEBUG):
raise
LOGERR(e)
sys.exit(1)
if __name__ == '__main__':
main()
ddgr-1.9/ddgr.1 0000664 0000000 0000000 00000021633 13707076421 0013321 0 ustar 00root root 0000000 0000000 .TH "DDGR" "1" "21 Jul 2020" "Version 1.9" "User Commands"
.SH NAME
ddgr \- DuckDuckGo from the terminal
.SH SYNOPSIS
.B ddgr [OPTIONS] [KEYWORD [KEYWORD ...]]
.SH DESCRIPTION
.B ddgr
is a command-line tool to search DuckDuckGo. \fBddgr\fR shows the title, URL and text context for each result. Results are fetched in pages. Keyboard shortcuts are available for page navigation. Results are indexed and a result URL can be opened in a browser using the index number. There is no configuration file as aliases serve the same purpose for this utility. Supports sequential searches in a single instance.
.PP
.B Features
.PP
* Fast and clean; custom color
* Designed for maximum readability at minimum space
* Instant answers (supported by DDG html version)
* Custom number of results per page
* Navigation, browser integration
* Search and option completion scripts (Bash, Fish, Zsh)
* DuckDuckGo Bangs (along with completion)
* Open the first result in browser (I'm Feeling Ducky)
* REPL for continuous searches
* Keywords (e.g. `filetype:mime`, `site:somesite.com`)
* Limit search by time, specify region, disable safe search
* HTTPS proxy support, optionally disable User Agent
* Do Not Track set by default
* Supports custom url handler script or cmdline utility
* Thoroughly documented, man page with examples
* Minimal dependencies
.SH OPTIONS
.TP
.BI "-h, --help"
Show help text and exit.
.TP
.BI "-n, --num=" N
Show N results per page (default 10). N must be between 0 and 25. N=0 disables fixed paging and shows actual number of results fetched per page.
.TP
.BI "-r, --reg=" REG
Region-specific search e.g. 'us-en' for US (default); visit https://duckduckgo.com/params.
.TP
.BI "-C, --nocolor"
Disable color output.
.TP
.BI "--colors=" COLORS
Set output colors. Refer to the \fBCOLORS\fR section below for details.
.TP
.BI "-j, --ducky"
Open the first result in a web browser; implies \fB--noprompt\fR. Feeling Ducky?
.TP
.BI "-t, --time=" SPAN
Time limit search [d=past day, w=past week, m=past month, y=past year] (default=any time).
.TP
.BI "-w, --site=" SITE
Search a site using DuckDuckGo.
.TP
.BI "-x, --expand"
Expand URLs instead of showing only the domain name (default).
.TP
.BI "-p, --proxy=" URI
Tunnel traffic through an HTTP proxy. \fIURI\fR is of the form \fI[http[s]://][user:pwd@]host[:port]\fR. The proxy server must support HTTP CONNECT tunneling and must not block port 443 for the relevant DuckDuckGo hosts. If a proxy is not explicitly given, the \fIhttps_proxy\fR or \fIHTTPS_PROXY\fR environment variable (if available) is used instead.
.TP
.BI "--unsafe"
Disable safe search.
.TP
.BI "--noua"
Disable user agent. Results are fetched faster.
.TP
.BI "--json"
Output in JSON format; implies \fB--noprompt\fR.
.TP
.BI "--gb, --gui-browser"
Open a bang directly in a GUI browser.
.TP
.BI "--np, --noprompt"
Perform search and exit; do not prompt for further interactions.
.TP
.BI "--url-handler=" UTIL
Custom script or command-line utility to open urls with.
.TP
.BI "--show-browser-logs"
Do not suppress browser output when opening result in browser; that is, connect stdout and stderr of the browser to ddgr's stdout and stderr instead of /dev/null. By default, browser output is suppressed (due to certain graphical browsers spewing messages to console) unless the \fBBROWSER\fR environment variable is a known text-based browser: elinks, links, lynx, w3m or www-browser.
.TP
.BI "-v, --version"
Show version number and exit.
.TP
.BI "-d, --debug"
Enable debugging.
.SH OMNIPROMPT KEYS
.TP
.BI "n, p, f"
Fetch the next, previous or first set of search results.
.TP
.BI "index"
Open the result corresponding to index in browser.
.TP
.BI o " [index|range|a ...]"
Open space-separated result indices, numeric ranges or all indices, if 'a' is specified, in the browser.
.TP
.BI O " [index|range|a ...]"
Works similar to key 'o', but tries to ignore text-based browsers (even if BROWSER is set) and open links in a GUI browser.
.TP
.BI d " keywords"
Initiate a new DuckDuckGo search for \fIkeywords\fR with original options. This key should be used to search omniprompt keys (including itself) and indices.
.TP
.BI "x"
Toggle url expansion.
.TP
.BI "c index"
Copy url to clipboard.
.TP
.BI "q, ^D, double Enter"
Exit ddgr.
.TP
.BI "?"
Show omniprompt help.
.TP
.BI *
Any other string initiates a new search with original options.
.SH COLORS
\fBddgr\fR allows you to customize the color scheme via a six-letter string, reminiscent of BSD \fBLSCOLORS\fR. The six letters represent the colors of
.IP - 2
indices
.PD 0 \" Change paragraph spacing to 0 in the list
.IP - 2
titles
.IP - 2
URLs
.IP - 2
metadata/publishing info
.IP - 2
abstracts
.IP - 2
prompts
.PD 1 \" Restore paragraph spacing
.TP
respectively. The six-letter string is passed in either as the argument to the \fB--colors\fR option, or as the value of the environment variable \fBDDGR_COLORS\fR.
.TP
We offer the following colors/styles:
.TS
tab(;) box;
l|l
-|-
l|l.
Letter;Color/Style
a;black
b;red
c;green
d;yellow
e;blue
f;magenta
g;cyan
h;white
i;bright black
j;bright red
k;bright green
l;bright yellow
m;bright blue
n;bright magenta
o;bright cyan
p;bright white
A-H;bold version of the lowercase-letter color
I-P;bold version of the lowercase-letter bright color
x;normal
X;bold
y;reverse video
Y;bold reverse video
.TE
.TP
.TP
The default colors string is \fIoCdgxy\fR, which stands for
.IP - 2
bright cyan indices
.PD 0 \" Change paragraph spacing to 0 in the list
.IP - 2
bold green titles
.IP - 2
yellow URLs
.IP - 2
cyan metadata/publishing info
.IP - 2
normal abstracts
.IP - 2
reverse video prompts
.PD 1 \" Restore paragraph spacing
.TP
Note that
.IP - 2
Bright colors (implemented as \\x1b[90m - \\x1b[97m) may not be available in all color-capable terminal emulators;
.IP - 2
Some terminal emulators draw bold text in bright colors instead;
.IP - 2
Some terminal emulators only distinguish between bold and bright colors via a default-off switch.
.TP
Please consult the manual of your terminal emulator as well as \fIhttps://en.wikipedia.org/wiki/ANSI_escape_code\fR for details.
.SH ENVIRONMENT
.TP
.BI BROWSER
Overrides the default browser. Ref:
.I http://docs.python.org/library/webbrowser.html
.TP
.BI DDGR_COLORS
Refer to the \fBCOLORS\fR section.
.TP
.BI DISABLE_PROMPT_COLOR
Force a plain omniprompt if you are facing issues with colors at the prompt.
.TP
.BI "HTTPS_PROXY, https_proxy"
Refer to the \fB--proxy\fR option.
.SH EXAMPLES
.PP
.IP 1. 4
DuckDuckGo \fBhello world\fR:
.PP
.EX
.IP
.B ddgr hello world
.EE
.PP
.IP 2. 4
\fBI'm Feeling Ducky\fR search:
.PP
.EX
.IP
.B ddgr -j lucky ducks
.EE
.PP
.IP 3. 4
\fBDuckDuckGo Bang\fR search 'hello world' in Wikipedia:
.PP
.EX
.IP
.B ddgr !w hello world
.B ddgr \\\\!w hello world // bash-specific, need to escape ! on bash
.EE
.PP
.IP "" 4
Bangs work at the omniprompt too. To look up bangs, visit https://duckduckgo.com/bang?#bangs-list.
.PP
.IP 4. 4
\fBBang alias\fR to fire from the cmdline, open results in a GUI browser and exit:
.PP
.EX
.IP
.B alias bang='ddgr --gb --np'
.IP
.B bang !w hello world
.B bang \\\\!w hello world // bash-specific, need to escape ! on bash
.EE
.PP
.IP 5. 4
\fBWebsite specific\fR search:
.PP
.EX
.IP
.B ddgr -w amazon.com digital camera
.EE
.PP
.IP "" 4
Site specific search continues at omniprompt.
.EE
.PP
.IP 6. 4
Search for a \fBspecific file type\fR:
.PP
.EX
.IP
.B ddgr instrumental filetype:mp3
.EE
.PP
.IP 7. 4
Fetch results on IPL cricket from \fBIndia\fR in \fBEnglish\fR:
.PP
.EX
.IP
.B ddgr -r in-en IPL cricket
.EE
.PP
.IP "" 4
To find your region parameter token visit https://duckduckgo.com/params.
.PP
.IP 8. 4
Search \fBquoted text\fR:
.PP
.EX
.IP
.B ddgr it\(rs's a \(rs\(dqbeautiful world\(rs\(dq in spring
.EE
.PP
.IP 9. 4
Show \fBcomplete urls\fR in search results (instead of only domain name):
.PP
.EX
.IP
.B ddgr -x ddgr
.EE
.PP
.IP 10. 4
Use a \fBcustom color scheme\fR, e.g., one warm color scheme designed for Solarized Dark:
.PP
.EX
.IP
.B ddgr --colors bjdxxy hello world
.IP
.B DDGR_COLORS=bjdxxy ddgr hello world
.EE
.PP
.IP 11. 4
Tunnel traffic through an \fBHTTPS proxy\fR, e.g., a local Privoxy instance listening on port 8118:
.PP
.EX
.IP
.B ddgr --proxy localhost:8118 hello world
.EE
.PP
.IP "" 4
By default the environment variable \fIhttps_proxy\fR (or \fIHTTPS_PROXY\fR) is used, if defined.
.EE
.PP
.IP 12. 4
Look up \fBn\fR, \fBp\fR, \fBo\fR, \fBO\fR, \fBq\fR, \fBd keywords\fR or a result index at the \fBomniprompt\fR: as the omniprompt recognizes these keys or index strings as commands, you need to prefix them with \fBd\fR, e.g.,
.PP
.EX
.PD 0
.IP
.B d n
.IP
.B d d keywords
.IP
.B d 1
.PD
.EE
.SH AUTHOR
Arun Prakash Jana
.SH HOME
.I https://github.com/jarun/ddgr
.SH REPORTING BUGS
.I https://github.com/jarun/ddgr/issues
.SH LICENSE
Copyright \(co 2016-2020 Arun Prakash Jana
.PP
License GPLv3+: GNU GPL version 3 or later .
.br
This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law.
ddgr-1.9/packagecore.yaml 0000664 0000000 0000000 00000002465 13707076421 0015451 0 ustar 00root root 0000000 0000000 name: ddgr
maintainer: Arun Prakash Jana
license: GPLv3
summary: DuckDuckGo from the terminal.
homepage: https://github.com/jarun/ddgr
commands:
install:
- make PREFIX="/usr" install DESTDIR="${BP_DESTDIR}"
packages:
# archlinux:
# builddeps:
# - make
# deps:
# - python
centos7.5:
builddeps:
- make
deps:
- python
centos7.6:
builddeps:
- make
deps:
- python
centos7.7:
builddeps:
- make
deps:
- python
centos8.0:
builddeps:
- make
deps:
- python3
commands:
precompile:
- dnf install python3
debian9:
builddeps:
- make
deps:
- python3
debian10:
builddeps:
- make
deps:
- python3
fedora31:
builddeps:
- make
deps:
- python3
fedora32:
builddeps:
- make
deps:
- python3
opensuse15.1:
builddeps:
- make
deps:
- python3
opensuse15.2:
builddeps:
- make
deps:
- python3
opensuse.tumbleweed:
builddeps:
- make
deps:
- python3
ubuntu16.04:
builddeps:
- make
deps:
- python3
ubuntu18.04:
builddeps:
- make
deps:
- python3
ubuntu20.04:
builddeps:
- make
deps:
- python3
ddgr-1.9/setup.py 0000664 0000000 0000000 00000003151 13707076421 0014024 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
import re
import os.path
import setuptools
import shutil
if os.path.isfile('ddgr'):
shutil.copyfile('ddgr', 'ddgr.py')
with open('ddgr.py', encoding='utf-8') as fp:
version = re.search(r'_VERSION_ = \'(.*?)\'', fp.read()).group(1)
with open('README.md', encoding='utf-8') as f:
long_description = f.read()
setuptools.setup(
name='ddgr',
version=version,
url='https://github.com/jarun/ddgr',
license='GPLv3',
license_file='LICENSE',
author='Arun Prakash Jana',
author_email='engineerarun@gmail.com',
description='DuckDuckGo from the terminal',
long_description=long_description,
long_description_content_type='text/markdown',
python_requires='>=3.5',
platforms=['any'],
py_modules=['ddgr'],
entry_points={
'console_scripts': [
'ddgr = ddgr:main',
],
},
classifiers=[
'Development Status :: 5 - Production/Stable',
'Environment :: Console',
'Intended Audience :: Developers',
'Intended Audience :: End Users/Desktop',
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Operating System :: OS Independent',
'Programming Language :: Python',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3 :: Only',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Topic :: Internet :: WWW/HTTP :: Indexing/Search',
'Topic :: Utilities',
],
)
tag.
#
#
#
#
#
# - Each title looks like
#
#
#
#
# result title
#
# file type (e.g. [PDF])
#
#
#
def __init__(self, offset=0):
html.parser.HTMLParser.__init__(self)
self.title = ''
self.url = ''
self.abstract = ''
self.filetype = ''
self.results = []
self.index = offset
self.textbuf = ''
self.click_result = ''
self.tag_annotations = {}
self.np_prev_button = ''
self.np_next_button = ''
self.npfound = False # First next params found
self.set_handlers_to('root')
self.vqd = '' # vqd returned from button, required for search query to get next set of results
# Tag handlers
@annotate_tag
def root_start(self, tag, attrs):
if tag == 'div':
if 'zci__result' in self.classes(attrs):
self.start_populating_textbuf()
return 'click_result'
if 'links_main' in self.classes(attrs):
# Initialize result field registers
self.title = ''
self.url = ''
self.abstract = ''
self.filetype = ''
self.set_handlers_to('result')
return 'result'
if 'nav-link' in self.classes(attrs):
self.set_handlers_to('input')
return 'input'
return ''
@retrieve_tag_annotation
def root_end(self, tag, annotation):
if annotation == 'click_result':
self.stop_populating_textbuf()
self.click_result = self.pop_textbuf()
self.set_handlers_to('root')
@annotate_tag
def result_start(self, tag, attrs):
if tag == 'h2' and 'result__title' in self.classes(attrs):
self.set_handlers_to('title')
return 'title'
if tag == 'a' and 'result__snippet' in self.classes(attrs) and 'href' in attrs:
self.start_populating_textbuf()
return 'abstract'
return ''
@retrieve_tag_annotation
def result_end(self, tag, annotation):
if annotation == 'abstract':
self.stop_populating_textbuf()
self.abstract = self.pop_textbuf()
elif annotation == 'result':
if self.url:
self.index += 1
result = Result(self.index, self.title, self.url, self.abstract, None)
self.results.append(result)
self.set_handlers_to('root')
@annotate_tag
def title_start(self, tag, attrs):
if tag == 'span':
# Print a space after the filetype indicator
self.start_populating_textbuf(lambda text: '[' + text + ']')
return 'title_filetype'
if tag == 'a' and 'href' in attrs:
# Skip 'News for', 'Images for' search links
if attrs['href'].startswith('/search'):
return ''
self.url = attrs['href']
try:
start = self.url.index('?q=') + len('?q=')
end = self.url.index('&sa=', start)
self.url = urllib.parse.unquote_plus(self.url[start:end])
except ValueError:
pass
self.start_populating_textbuf()
return 'title_link'
return ''
@retrieve_tag_annotation
def title_end(self, tag, annotation):
if annotation == 'title_filetype':
self.stop_populating_textbuf()
self.filetype = self.pop_textbuf()
self.start_populating_textbuf()
elif annotation == 'title_link':
self.stop_populating_textbuf()
self.title = self.pop_textbuf()
if self.filetype != '':
self.title = self.filetype + self.title
elif annotation == 'title':
self.set_handlers_to('result')
@annotate_tag
def abstract_start(self, tag, attrs):
if tag == 'span' and 'st' in self.classes(attrs):
self.start_populating_textbuf()
return 'abstract_text'
return ''
@retrieve_tag_annotation
def abstract_end(self, tag, annotation):
if annotation == 'abstract_text':
self.stop_populating_textbuf()
self.abstract = self.pop_textbuf()
elif annotation == 'abstract':
self.set_handlers_to('result')
@annotate_tag
def input_start(self, tag, attrs):
if tag == 'input' and 'name' in attrs:
if attrs['name'] == 'nextParams':
# The previous button always shows before next button
# If there's only 1 button (page 1), it's the next button
if self.npfound is True:
self.np_prev_button = self.np_next_button
else:
self.npfound = True
self.np_next_button = attrs['value']
return
if attrs['name'] == 'vqd' and attrs['value'] != '':
# vqd required to be passed for next/previous search page
self.vqd = attrs['value']
return
@retrieve_tag_annotation
def input_end(self, tag, annotation):
return
# Generic methods
# Set handle_starttag to SCOPE_start, and handle_endtag to SCOPE_end.
def set_handlers_to(self, scope):
self.handle_starttag = getattr(self, scope + '_start')
self.handle_endtag = getattr(self, scope + '_end')
def insert_annotation(self, tag, annotation):
if tag not in self.tag_annotations:
self.tag_annotations[tag] = []
self.tag_annotations[tag].append(annotation)
def start_populating_textbuf(self, data_transformer=None):
if data_transformer is None:
# Record data verbatim
self.handle_data = self.record_data
else:
def record_transformed_data(data):
self.textbuf += data_transformer(data)
self.handle_data = record_transformed_data
self.handle_entityref = self.record_entityref
self.handle_charref = self.record_charref
def pop_textbuf(self):
text = self.textbuf
self.textbuf = ''
return text
def stop_populating_textbuf(self):
self.handle_data = lambda data: None
self.handle_entityref = lambda ref: None
self.handle_charref = lambda ref: None
def record_data(self, data):
self.textbuf += data
def record_entityref(self, ref):
try:
self.textbuf += chr(html.entities.name2codepoint[ref])
except KeyError:
# Entity name not found; most likely rather sloppy HTML
# where a literal ampersand is not escaped; For instance,
# containing the following tag
#
#
expected market return s&p 500
# # where &p is interpreted by HTMLParser as an entity (this # behaviour seems to be specific to Python 2.7). self.textbuf += '&' + ref def record_charref(self, ref): if ref.startswith('x'): char = chr(int(ref[1:], 16)) else: char = chr(int(ref)) self.textbuf += char @staticmethod def classes(attrs): """Get tag's classes from its attribute dict.""" return attrs.get('class', '').split() def error(self, message): raise NotImplementedError("subclasses of ParserBase must override error()") Colors = collections.namedtuple('Colors', 'index, title, url, metadata, abstract, prompt, reset') class Result: """ Container for one search result, with output helpers. Parameters ---------- index : int or str title : str url : str abstract : str metadata : str, optional Only applicable to DuckDuckGo News results, with publisher name and publishing time. Attributes ---------- index : str title : str url : str abstract : str metadata : str or None Class Variables --------------- colors : str Methods ------- print() jsonizable_object() urltable() """ # Class variables colors = None urlexpand = False def __init__(self, index, title, url, abstract, metadata=None): index = str(index) self.index = index self.title = title self.url = url self.abstract = abstract self.metadata = metadata self._urltable = {index: url} def _print_title_and_url(self, index, title, url): indent = INDENT - 2 colors = self.colors if not self.urlexpand: url = '[' + urllib.parse.urlparse(url).netloc + ']' if colors: # Adjust index to print result index clearly print(" %s%-*s%s" % (colors.index, indent, index + '.', colors.reset), end='') if not self.urlexpand: print(' ' + colors.title + title + colors.reset + ' ' + colors.url + url + colors.reset) else: print(' ' + colors.title + title + colors.reset) print(' ' * (INDENT) + colors.url + url + colors.reset) else: if self.urlexpand: print(' %-*s %s' % (indent, index + '.', title)) print(' %s%s' % (' ' * (indent + 1), url)) else: print(' %-*s %s %s' % (indent, index + '.', title, url)) def _print_metadata_and_abstract(self, abstract, metadata=None): colors = self.colors try: columns, _ = os.get_terminal_size() except OSError: columns = 0 if metadata: if colors: print(' ' * INDENT + colors.metadata + metadata + colors.reset) else: print(' ' * INDENT + metadata) if colors: print(colors.abstract, end='') if columns > INDENT + 1: # Try to fill to columns fillwidth = columns - INDENT - 1 for line in textwrap.wrap(abstract.replace('\n', ''), width=fillwidth): print('%s%s' % (' ' * INDENT, line)) print('') else: print('%s\n' % abstract.replace('\n', ' ')) if colors: print(colors.reset, end='') def print(self): """Print the result entry.""" self._print_title_and_url(self.index, self.title, self.url) self._print_metadata_and_abstract(self.abstract, metadata=self.metadata) def print_paginated(self, display_index): """Print the result entry with custom index.""" self._print_title_and_url(display_index, self.title, self.url) self._print_metadata_and_abstract(self.abstract, metadata=self.metadata) def jsonizable_object(self): """Return a JSON-serializable dict representing the result entry.""" obj = { 'title': self.title, 'url': self.url, 'abstract': self.abstract } if self.metadata: obj['metadata'] = self.metadata return obj def urltable(self): """Return a index-to-URL table for the current result. Normally, the table contains only a single entry, but when the result contains sitelinks, all sitelinks are included in this table. Returns ------- dict A dict mapping indices (strs) to URLs (also strs). """ return self._urltable class DdgCmdException(Exception): pass class NoKeywordsException(DdgCmdException): pass def require_keywords(method): # Require keywords to be set before we run a DdgCmd method. If # no keywords have been set, raise a NoKeywordsException. @functools.wraps(method) def enforced_method(self, *args, **kwargs): if not self.keywords: raise NoKeywordsException('No keywords.') method(self, *args, **kwargs) return enforced_method def no_argument(method): # Normalize a do_* method of DdgCmd that takes no argument to # one that takes an arg, but issue a warning when an nonempty # argument is given. @functools.wraps(method) def enforced_method(self, arg): if arg: method_name = arg.__name__ command_name = method_name[3:] if method_name.startswith('do_') else method_name LOGGER.warning("Argument to the '%s' command ignored.", command_name) method(self) return enforced_method class DdgCmd: """ Command line interpreter and executor class for ddgr. Inspired by PSL cmd.Cmd. Parameters ---------- opts : argparse.Namespace Options and/or arguments. Attributes ---------- options : argparse.Namespace Options that are currently in effect. Read-only attribute. keywords : str or list or strs Current keywords. Read-only attribute Methods ------- fetch() display_results(prelude='\n', json_output=False) fetch_and_display(prelude='\n', json_output=False) read_next_command() help() cmdloop() """ def __init__(self, opts, ua): super().__init__() self.cmd = '' self.index = 0 self._opts = opts self._ddg_url = DdgUrl(opts) proxy = opts.proxy if hasattr(opts, 'proxy') else None self._conn = DdgConnection(proxy=proxy, ua=ua) self.results = [] self._urltable = {} colors = self.colors message = 'ddgr (? for help)' self.prompt = ((colors.prompt + message + colors.reset + ' ') if (colors and os.getenv('DISABLE_PROMPT_COLOR') is None) else (message + ': ')) @property def options(self): """Current options.""" return self._opts @property def keywords(self): """Current keywords.""" return self._ddg_url.keywords @require_keywords def fetch(self, json_output=False): """Fetch a page and parse for results. Results are stored in ``self.results``. Parameters ---------- json_output : bool, optional Whether to dump results in JSON format. Default is False. Raises ------ DDGConnectionError See Also -------- fetch_and_display """ # This method also sets self._urltable. page = self._conn.fetch_page(self._ddg_url) if page is None: return if LOGGER.isEnabledFor(logging.DEBUG): fd, tmpfile = tempfile.mkstemp(prefix='ddgr-response-') os.close(fd) with open(tmpfile, 'w', encoding='utf-8') as fp: fp.write(page) LOGDBG("Response body written to '%s'.", tmpfile) if self._opts.num: _index = len(self._urltable) else: _index = 0 self._urltable = {} parser = DdgParser(offset=_index) parser.feed(page) if self._opts.num: self.results.extend(parser.results) else: self.results = parser.results for r in parser.results: self._urltable.update(r.urltable()) self._ddg_url.np_prev = parser.np_prev_button self._ddg_url.np_next = parser.np_next_button self._ddg_url.vqd = parser.vqd # Show instant answer if self.index == 0 and parser.click_result and not json_output: if self.colors: print(self.colors.abstract) try: columns, _ = os.get_terminal_size() except OSError: columns = 0 fillwidth = columns - INDENT for line in textwrap.wrap(parser.click_result.strip(), width=fillwidth): print('%s%s' % (' ' * INDENT, line)) if self.colors: print(self.colors.reset, end='') LOGDBG('Prev nextParams: %s', self._ddg_url.np_prev) LOGDBG('Next nextParams: %s', self._ddg_url.np_next) LOGDBG('VQD: %s', self._ddg_url.vqd) self._ddg_url.update_num(len(parser.results)) @require_keywords def display_results(self, prelude='\n', json_output=False): """Display results stored in ``self.results``. Parameters ---------- See `fetch_and_display`. """ if self._opts.num: results = self.results[self.index:(self.index + self._opts.num)] else: results = self.results if json_output: # JSON output results_object = [r.jsonizable_object() for r in results] print(json.dumps(results_object, indent=2, sort_keys=True, ensure_ascii=False)) elif not results: print('No results.', file=sys.stderr) elif self._opts.num: # Paginated output sys.stderr.write(prelude) for i, r in enumerate(results): r.print_paginated(str(i + 1)) else: # Regular output sys.stderr.write(prelude) for r in results: r.print() @require_keywords def fetch_and_display(self, prelude='\n', json_output=False): """Fetch a page and display results. Results are stored in ``self.results``. Parameters ---------- prelude : str, optional A string that is written to stderr before showing actual results, usually serving as a separator. Default is an empty line. json_output : bool, optional Whether to dump results in JSON format. Default is False. Raises ------ DDGConnectionError See Also -------- fetch display_results """ self.fetch() self.display_results(prelude=prelude, json_output=json_output) def read_next_command(self): """Show omniprompt and read user command line. Command line is always stripped, and each consecutive group of whitespace is replaced with a single space character. If the command line is empty after stripping, when ignore it and keep reading. Exit with status 0 if we get EOF or an empty line (pre-strip, that is, a raw