')
status('%s save finished' % p)
save_htm = multisave_htm = multisave_html = save_html
visidata-1.5.2/visidata/loaders/postgres.py 0000660 0001750 0001750 00000006122 13416252050 021465 0 ustar kefala kefala 0000000 0000000 from visidata import *
def codeToType(type_code, colname):
import psycopg2
try:
tname = psycopg2._psycopg.string_types[type_code].name
if 'INTEGER' in tname:
return int
if 'STRING' in tname:
return str
except KeyError:
status('unknown postgres type_code %s for %s' % (type_code, colname))
return anytype
def openurl_postgres(url, filetype=None):
import psycopg2
dbname = url.path[1:]
conn = psycopg2.connect(
user=url.username,
dbname=dbname,
host=url.hostname,
port=url.port,
password=url.password)
return PgTablesSheet(dbname+"_tables", sql=SQL(conn))
class SQL:
def __init__(self, conn):
self.conn = conn
def cur(self, qstr):
randomname = ''.join(random.choice(string.ascii_uppercase) for _ in range(6))
cur = self.conn.cursor(randomname)
cur.execute(qstr)
return cur
@asyncthread
def query_async(self, qstr, callback=None):
with self.cur(qstr) as cur:
callback(cur)
cur.close()
def cursorToColumns(cur):
cols = []
for i, coldesc in enumerate(cur.description):
c = ColumnItem(coldesc.name, i, type=codeToType(coldesc.type_code, coldesc.name))
cols.append(c)
return cols
# rowdef: (table_name, ncols)
class PgTablesSheet(Sheet):
rowtype = 'tables'
def reload(self):
qstr = "SELECT table_name, COUNT(column_name) AS ncols FROM information_schema.columns WHERE table_schema = 'public' GROUP BY table_name"
with self.sql.cur(qstr) as cur:
self.nrowsPerTable = {}
self.rows = []
# try to get first row to make cur.description available
r = cur.fetchone()
if r:
self.addRow(r)
self.columns = cursorToColumns(cur)
self.setKeys(self.columns[0:1]) # table_name is the key
self.addColumn(Column('nrows', type=int, getter=lambda col,row: col.sheet.getRowCount(row[0])))
for r in cur:
self.addRow(r)
def setRowCount(self, cur):
result = cur.fetchall()
tablename = result[0][0]
self.nrowsPerTable[tablename] = result[0][1]
def getRowCount(self, tablename):
if tablename not in self.nrowsPerTable:
thread = self.sql.query_async("SELECT '%s', COUNT(*) FROM %s" % (tablename, tablename), callback=self.setRowCount)
self.nrowsPerTable[tablename] = thread
return self.nrowsPerTable[tablename]
PgTablesSheet.addCommand(ENTER, 'dive-row', 'vd.push(PgTable(name+"."+cursorRow[0], source=cursorRow[0], sql=sql))')
# rowdef: tuple of values as returned by fetchone()
class PgTable(Sheet):
@asyncthread
def reload(self):
with self.sql.cur("SELECT * FROM " + self.source) as cur:
self.rows = []
r = cur.fetchone()
if r:
self.addRow(r)
self.columns = cursorToColumns(cur)
for r in cur:
self.addRow(r)
addGlobals(globals())
visidata-1.5.2/visidata/loaders/spss.py 0000660 0001750 0001750 00000001232 13416252050 020604 0 ustar kefala kefala 0000000 0000000 from visidata import *
def open_spss(p):
return SpssSheet(p.name, source=p)
open_sav = open_spss
class SpssSheet(Sheet):
@asyncthread
def reload(self):
import savReaderWriter
self.rdr = savReaderWriter.SavReader(self.source.resolve())
with self.rdr as reader:
self.columns = []
for i, vname in enumerate(reader.varNames):
vtype = float if reader.varTypes[vname] == 0 else str
self.addColumn(ColumnItem(vname.decode('utf-8'), i, type=vtype))
self.rows = []
for r in Progress(reader, total=reader.shape.nrows):
self.rows.append(r)
visidata-1.5.2/visidata/loaders/csv.py 0000660 0001750 0001750 00000006037 13416500243 020417 0 ustar kefala kefala 0000000 0000000
from visidata import *
import csv
replayableOption('csv_dialect', 'excel', 'dialect passed to csv.reader')
replayableOption('csv_delimiter', ',', 'delimiter passed to csv.reader')
replayableOption('csv_quotechar', '"', 'quotechar passed to csv.reader')
replayableOption('csv_skipinitialspace', True, 'skipinitialspace passed to csv.reader')
replayableOption('csv_escapechar', None, 'escapechar passed to csv.reader')
replayableOption('safety_first', False, 'sanitize input/output to handle edge cases, with a performance cost')
csv.field_size_limit(sys.maxsize)
options_num_first_rows = 10
def open_csv(p):
return CsvSheet(p.name, source=p)
def wrappedNext(rdr):
try:
return next(rdr)
except csv.Error as e:
return ['[csv.Error: %s]' % e]
def removeNulls(fp):
for line in fp:
yield line.replace('\0', '')
class CsvSheet(Sheet):
_rowtype = list
# _coltype = ColumnItem
@asyncthread
def reload(self):
load_csv(self)
def newRow(self):
return [None]*len(self.columns)
def csvoptions():
return options('csv_')
def load_csv(vs):
'Convert from CSV, first handling header row specially.'
with vs.source.open_text() as fp:
for i in range(options.skip):
wrappedNext(fp) # discard initial lines
if options.safety_first:
rdr = csv.reader(removeNulls(fp), **csvoptions())
else:
rdr = csv.reader(fp, **csvoptions())
vs.rows = []
# headers first, to setup columns before adding rows
headers = [wrappedNext(rdr) for i in range(int(options.header))]
if headers:
# columns ideally reflect the max number of fields over all rows
vs.columns = ArrayNamedColumns('\\n'.join(x) for x in zip(*headers))
else:
r = wrappedNext(rdr)
vs.addRow(r)
vs.columns = ArrayColumns(len(vs.rows[0]))
if not vs.columns:
vs.columns = [ColumnItem(0)]
vs.recalc() # make columns usable
with Progress(total=vs.source.filesize) as prog:
try:
samplelen = 0
for i in range(options_num_first_rows): # for progress below
row = wrappedNext(rdr)
vs.addRow(row)
samplelen += sum(len(x) for x in row)
samplelen //= options_num_first_rows # avg len of first n rows
while True:
vs.addRow(wrappedNext(rdr))
prog.addProgress(samplelen)
except StopIteration:
pass # as expected
vs.recalc()
return vs
@asyncthread
def save_csv(p, sheet):
'Save as single CSV file, handling column names as first line.'
with p.open_text(mode='w') as fp:
cw = csv.writer(fp, **csvoptions())
colnames = [col.name for col in sheet.visibleCols]
if ''.join(colnames):
cw.writerow(colnames)
for r in Progress(sheet.rows, 'saving'):
cw.writerow([col.getDisplayValue(r) for col in sheet.visibleCols])
visidata-1.5.2/visidata/loaders/pcap.py 0000660 0001750 0001750 00000032454 13416252050 020551 0 ustar kefala kefala 0000000 0000000 import collections
import ipaddress
from visidata import *
option('pcap_internet', 'n', '(y/s/n) if save_dot includes all internet hosts separately (y), combined (s), or does not include the internet (n)')
protocols = collections.defaultdict(dict) # ['ethernet'] = {[6] -> 'IP'}
_flags = collections.defaultdict(dict) # ['tcp'] = {[4] -> 'FIN'}
url_oui = 'https://visidata.org/data/wireshark-oui.tsv'
url_iana = 'https://visidata.org/data/iana-ports.tsv'
oui = {} # [macprefix (like '01:02:dd:0')] -> 'manufacturer'
services = {} # [('tcp', 25)] -> 'smtp'
def manuf(mac):
return oui.get(mac[:13]) or oui.get(mac[:10]) or oui.get(mac[:8])
def macaddr(addrbytes):
mac = ':'.join('%02x' % b for b in addrbytes)
return mac
def macmanuf(mac):
manuf = oui.get(mac[:13])
if manuf:
return manuf + mac[13:]
manuf = oui.get(mac[:10])
if manuf:
return manuf + mac[10:]
manuf = oui.get(mac[:8])
if manuf:
return manuf + mac[8:]
return mac
def norm_host(host):
if not host:
return None
srcmac = str(host.macaddr)
if srcmac == 'ff:ff:ff:ff:ff:ff': return None
srcip = str(host.ipaddr)
if srcip == '0.0.0.0' or srcip == '::': return None
if srcip == '255.255.255.255': return None
if host.ipaddr:
if host.ipaddr.is_global:
opt = options.pcap_internet
if opt == 'n':
return None
elif opt == 's':
return "internet"
if host.ipaddr.is_multicast:
# include in multicast (minus dns?)
return 'multicast'
names = [host.hostname, host.ipaddr, macmanuf(host.macaddr)]
return '\\n'.join(str(x) for x in names if x)
def FlagGetter(flagfield):
def flags_func(fl):
return ' '.join([flagname for f, flagname in _flags[flagfield].items() if fl & f])
return flags_func
def init_pcap():
if protocols: # already init'ed
return
global dpkt, dnslib
import dpkt
import dnslib
load_consts(protocols['ethernet'], dpkt.ethernet, 'ETH_TYPE_')
load_consts(protocols['ip'], dpkt.ip, 'IP_PROTO_')
load_consts(_flags['ip_tos'], dpkt.ip, 'IP_TOS_')
load_consts(protocols['icmp'], dpkt.icmp, 'ICMP_')
load_consts(_flags['tcp'], dpkt.tcp, 'TH_')
load_oui(url_oui)
load_iana(url_iana)
def read_pcap(f):
try:
return dpkt.pcapng.Reader(f.open_bytes())
except ValueError:
return dpkt.pcap.Reader(f.open_bytes())
@asyncthread
def load_oui(url):
vsoui = open_tsv(urlcache(url, 30*days))
vsoui.reload_sync()
for r in vsoui.rows:
if r.prefix.endswith('/36'): prefix = r.prefix[:13]
elif r.prefix.endswith('/28'): prefix = r.prefix[:10]
else: prefix = r.prefix[:8]
try:
oui[prefix.lower()] = r.shortname
except Exception as e:
exceptionCaught(e)
@asyncthread
def load_iana(url):
ports_tsv = open_tsv(urlcache(url, 30*days))
ports_tsv.reload_sync()
for r in ports_tsv.rows:
try:
services[(r.transport, int(r.port))] = r.service
except Exception as e:
exceptionCaught(e)
class Host:
dns = {} # [ipstr] -> dnsname
hosts = {} # [macaddr] -> { [ipaddr] -> Host }
@classmethod
def get_host(cls, pkt, field='src'):
mac = macaddr(getattr(pkt, field))
machosts = cls.hosts.get(mac, None)
if not machosts:
machosts = cls.hosts[mac] = {}
ipraw = getattrdeep(pkt, 'ip.'+field, None)
if ipraw is not None:
ip = ipaddress.ip_address(ipraw)
if ip not in machosts:
machosts[ip] = Host(mac, ip)
return machosts[ip]
else:
if machosts:
return list(machosts.values())[0]
return Host(mac, None)
@classmethod
def get_by_ip(cls, ip):
'Returns Host instance for the given ip address.'
ret = cls.hosts_by_ip.get(ip)
if ret is None:
ret = cls.hosts_by_ip[ip] = [Host(ip)]
return ret
def __init__(self, mac, ip):
self.ipaddr = ip
self.macaddr = mac
self.mac_manuf = None
def __str__(self):
return str(self.hostname or self.ipaddr or macmanuf(self.macaddr))
def __lt__(self, x):
if isinstance(x, Host):
return str(self.ipaddr) < str(x.ipaddr)
return True
@property
def hostname(self):
return Host.dns.get(str(self.ipaddr))
def load_consts(outdict, module, attrprefix):
for k in dir(module):
if k.startswith(attrprefix):
v = getattr(module, k)
outdict[v] = k[len(attrprefix):]
def getTuple(pkt):
if getattrdeep(pkt, 'ip.tcp', None):
tup = ('tcp', Host.get_host(pkt, 'src'), pkt.ip.tcp.sport, Host.get_host(pkt, 'dst'), pkt.ip.tcp.dport)
elif getattrdeep(pkt, 'ip.udp', None):
tup = ('udp', Host.get_host(pkt, 'src'), pkt.ip.udp.sport, Host.get_host(pkt, 'dst'), pkt.ip.udp.dport)
else:
return None
a,b,c,d,e = tup
if b > d:
return a,d,e,b,c # swap src/sport and dst/dport
else:
return tup
def getService(tup):
if not tup: return
transport, _, sport, _, dport = tup
if (transport, dport) in services:
return services.get((transport, dport))
if (transport, sport) in services:
return services.get((transport, sport))
def get_transport(pkt):
ret = 'ether'
if getattr(pkt, 'arp', None):
return 'arp'
if getattr(pkt, 'ip', None):
ret = 'ip'
if getattr(pkt.ip, 'tcp', None):
ret = 'tcp'
elif getattr(pkt.ip, 'udp', None):
ret = 'udp'
elif getattr(pkt.ip, 'icmp', None):
ret = 'icmp'
if getattr(pkt, 'ip6', None):
ret = 'ipv6'
if getattr(pkt.ip6, 'tcp', None):
ret = 'tcp'
elif getattr(pkt.ip6, 'udp', None):
ret = 'udp'
elif getattr(pkt.ip6, 'icmp6', None):
ret = 'icmpv6'
return ret
def get_port(pkt, field='sport'):
return getattrdeep(pkt, 'ip.tcp.'+field, None) or getattrdeep(pkt, 'ip.udp.'+field, None)
class EtherSheet(Sheet):
'Layer 2 (ethernet) packets'
rowtype = 'packets'
columns = [
ColumnAttr('timestamp', type=date, fmtstr="%H:%M:%S.%f"),
Column('ether_manuf', getter=lambda col,row: mac_manuf(macaddr(row.src))),
Column('ether_src', getter=lambda col,row: macaddr(row.src), width=6),
Column('ether_dst', getter=lambda col,row: macaddr(row.dst), width=6),
ColumnAttr('ether_data', 'data', type=len, width=0),
]
class IPSheet(Sheet):
rowtype = 'packets'
columns = [
ColumnAttr('timestamp', type=date, fmtstr="%H:%M:%S.%f"),
ColumnAttr('ip', width=0),
Column('ip_src', width=14, getter=lambda col,row: ipaddress.ip_address(row.ip.src)),
Column('ip_dst', width=14, getter=lambda col,row: ipaddress.ip_address(row.ip.dst)),
ColumnAttr('ip_hdrlen', 'ip.hl', width=0, helpstr="IPv4 Header Length"),
ColumnAttr('ip_proto', 'ip.p', type=lambda v: protocols['ip'].get(v), width=8, helpstr="IPv4 Protocol"),
ColumnAttr('ip_id', 'ip.id', width=0, helpstr="IPv4 Identification"),
ColumnAttr('ip_rf', 'ip.rf', width=0, helpstr="IPv4 Reserved Flag (Evil Bit)"),
ColumnAttr('ip_df', 'ip.df', width=0, helpstr="IPv4 Don't Fragment flag"),
ColumnAttr('ip_mf', 'ip.mf', width=0, helpstr="IPv4 More Fragments flag"),
ColumnAttr('ip_tos', 'ip.tos', width=0, type=FlagGetter('ip_tos'), helpstr="IPv4 Type of Service"),
ColumnAttr('ip_ttl', 'ip.ttl', width=0, helpstr="IPv4 Time To Live"),
ColumnAttr('ip_ver', 'ip.v', width=0, helpstr="IPv4 Version"),
]
def reload(self):
self.rows = []
for pkt in Progress(self.source.rows):
if getattr(pkt, 'ip', None):
self.addRow(pkt)
class TCPSheet(IPSheet):
columns = IPSheet.columns + [
ColumnAttr('tcp_srcport', 'ip.tcp.sport', type=int, width=8, helpstr="TCP Source Port"),
ColumnAttr('tcp_dstport', 'ip.tcp.dport', type=int, width=8, helpstr="TCP Dest Port"),
ColumnAttr('tcp_opts', 'ip.tcp.opts', width=0),
ColumnAttr('tcp_flags', 'ip.tcp.flags', type=FlagGetter('tcp'), helpstr="TCP Flags"),
]
def reload(self):
self.rows = []
for pkt in Progress(self.source.rows):
if getattrdeep(pkt, 'ip.tcp', None):
self.addRow(pkt)
class UDPSheet(IPSheet):
columns = IPSheet.columns + [
ColumnAttr('udp_srcport', 'ip.udp.sport', type=int, width=8, helpstr="UDP Source Port"),
ColumnAttr('udp_dstport', 'ip.udp.dport', type=int, width=8, helpstr="UDP Dest Port"),
ColumnAttr('ip.udp.data', type=len, width=0),
ColumnAttr('ip.udp.ulen', type=int, width=0),
]
def reload(self):
self.rows = []
for pkt in Progress(self.source.rows):
if getattrdeep(pkt, 'ip.udp', None):
self.addRow(pkt)
class PcapSheet(Sheet):
rowtype = 'packets'
columns = [
ColumnAttr('timestamp', type=date, fmtstr="%H:%M:%S.%f"),
Column('transport', type=get_transport, width=5),
Column('srcmanuf', getter=lambda col,row: manuf(macaddr(row.src))),
Column('srchost', getter=lambda col,row: row.srchost),
Column('srcport', type=int, getter=lambda col,row: get_port(row, 'sport')),
Column('dstmanuf', getter=lambda col,row: manuf(macaddr(row.dst))),
Column('dsthost', getter=lambda col,row: row.dsthost),
Column('dstport', type=int, getter=lambda col,row: get_port(row, 'dport')),
ColumnAttr('ether_proto', 'type', type=lambda v: protocols['ethernet'].get(v), width=0),
ColumnAttr('tcp_flags', 'ip.tcp.flags', type=FlagGetter('tcp'), helpstr="TCP Flags"),
Column('service', getter=lambda col,row: getService(getTuple(row))),
ColumnAttr('data', type=len),
ColumnAttr('ip.len', type=int),
ColumnAttr('tcp', 'ip.tcp', width=4, type=len),
ColumnAttr('udp', 'ip.udp', width=4, type=len),
ColumnAttr('icmp', 'ip.icmp', width=4, type=len),
ColumnAttr('dns', width=4),
]
@asyncthread
def reload(self):
init_pcap()
self.pcap = read_pcap(self.source)
self.rows = []
with Progress(total=self.source.filesize) as prog:
for ts, buf in self.pcap:
eth = dpkt.ethernet.Ethernet(buf)
self.addRow(eth)
prog.addProgress(len(buf))
eth.timestamp = ts
if not getattr(eth, 'ip', None):
eth.ip = getattr(eth, 'ip6', None)
eth.dns = try_apply(lambda eth: dnslib.DNSRecord.parse(eth.ip.udp.data), eth)
if eth.dns:
for rr in eth.dns.rr:
Host.dns[str(rr.rdata)] = str(rr.rname)
eth.srchost = Host.get_host(eth, 'src')
eth.dsthost = Host.get_host(eth, 'dst')
PcapSheet.addCommand('W', 'flows', 'vd.push(PcapFlowsSheet(sheet.name+"_flows", source=sheet))')
PcapSheet.addCommand('2', 'l2-packet', 'vd.push(IPSheet("L2packets", source=sheet))')
PcapSheet.addCommand('3', 'l3-packet', 'vd.push(TCPSheet("L3packets", source=sheet))')
flowtype = collections.namedtuple('flow', 'packets transport src sport dst dport'.split())
class PcapFlowsSheet(Sheet):
rowtype = 'netflows' # rowdef: flowtype
_rowtype = flowtype
columns = [
ColumnAttr('transport'),
Column('src', getter=lambda col,row: row.src),
ColumnAttr('sport', type=int),
Column('dst', getter=lambda col,row: row.dst),
ColumnAttr('dport', type=int),
Column('service', width=8, getter=lambda col,row: getService(getTuple(row.packets[0]))),
ColumnAttr('packets', type=len),
Column('connect_latency_ms', type=float, getter=lambda col,row: col.sheet.latency[getTuple(row.packets[0])]),
]
@asyncthread
def reload(self):
self.rows = []
self.flows = {}
self.latency = {} # [flowtuple] -> float ms of latency
self.syntimes = {} # [flowtuple] -> timestamp of SYN
flags = FlagGetter('tcp')
for pkt in Progress(self.source.rows):
tup = getTuple(pkt)
if tup:
flowpkts = self.flows.get(tup)
if flowpkts is None:
flowpkts = self.flows[tup] = []
self.addRow(flowtype(flowpkts, *tup))
flowpkts.append(pkt)
if not getattr(pkt.ip, 'tcp', None):
continue
tcpfl = flags(pkt.ip.tcp.flags)
if 'SYN' in tcpfl:
if 'ACK' in tcpfl:
if tup in self.syntimes:
self.latency[tup] = (pkt.timestamp - self.syntimes[tup])*1000
else:
self.syntimes[tup] = pkt.timestamp
PcapFlowsSheet.addCommand(ENTER, 'dive-row', 'vd.push(PcapSheet("%s_packets"%flowname(cursorRow), rows=cursorRow.packets))')
def flowname(flow):
return '%s_%s:%s-%s:%s' % (flow.transport, flow.src, flow.sport, flow.dst, flow.dport)
def try_apply(func, *args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
pass
def open_pcap(p):
return PcapSheet(p.name, source=p)
open_cap = open_pcap
open_pcapng = open_pcap
open_ntar = open_pcap
visidata-1.5.2/visidata/loaders/tsv.py 0000660 0001750 0001750 00000012557 13416252050 020444 0 ustar kefala kefala 0000000 0000000 import os
import contextlib
import itertools
import collections
from visidata import asyncthread, options, Progress, status, ColumnItem, Sheet, FileExistsError, getType, exceptionCaught
from visidata.namedlist import namedlist
def getlines(fp, maxlines=None):
i = 0
while True:
if maxlines is not None and i >= maxlines:
break
try:
L = next(fp)
except StopIteration:
break
L = L.rstrip('\n')
if L:
yield L
i += 1
def open_tsv(p):
return TsvSheet(p.name, source=p)
# rowdef: namedlist
class TsvSheet(Sheet):
_rowtype = None
@asyncthread
def reload(self):
self.reload_sync()
def reload_sync(self):
'Perform synchronous loading of TSV file, discarding header lines.'
header_lines = options.get('header', self)
delim = options.get('delimiter', self)
with self.source.open_text() as fp:
# get one line anyway to determine number of columns
lines = list(getlines(fp, int(header_lines) or 1))
headers = [L.split(delim) for L in lines]
if header_lines <= 0:
self.columns = [ColumnItem('', i) for i in range(len(headers[0]))]
else:
self.columns = [
ColumnItem('\\n'.join(x), i)
for i, x in enumerate(zip(*headers[:header_lines]))
]
lines = lines[header_lines:] # in case of header_lines == 0
self._rowtype = namedlist('tsvobj', [c.name for c in self.columns])
self.recalc()
self.rows = []
with Progress(total=self.source.filesize) as prog:
for L in itertools.chain(lines, getlines(fp)):
row = L.split(delim)
ncols = self._rowtype.length() # current number of cols
if len(row) > ncols:
# add unnamed columns to the type not found in the header
newcols = [ColumnItem('', len(row)+i, width=8) for i in range(len(row)-ncols)]
self._rowtype = namedlist(self._rowtype.__name__, list(self._rowtype._fields) + ['_' for c in newcols])
for c in newcols:
self.addColumn(c)
elif len(row) < ncols:
# extend rows that are missing entries
row.extend([None]*(ncols-len(row)))
self.addRow(self._rowtype(row))
prog.addProgress(len(L))
def newRow(self):
return self._rowtype()
def tsv_trdict(vs):
'returns string.translate dictionary for replacing tabs and newlines'
if options.safety_first:
delim = options.get('delimiter', vs)
return {ord(delim): options.get('tsv_safe_tab', vs), # \t
10: options.get('tsv_safe_newline', vs), # \n
13: options.get('tsv_safe_newline', vs), # \r
}
return {}
def save_tsv_header(p, vs):
'Write tsv header for Sheet `vs` to Path `p`.'
trdict = tsv_trdict(vs)
delim = options.delimiter
with p.open_text(mode='w') as fp:
colhdr = delim.join(col.name.translate(trdict) for col in vs.visibleCols) + '\n'
if colhdr.strip(): # is anything but whitespace
fp.write(colhdr)
def genAllValues(rows, cols, trdict={}, format=True):
transformers = collections.OrderedDict() # list of transformers for each column in order
for col in cols:
transformers[col] = [ col.type ]
if format:
transformers[col].append(
lambda v,fmtfunc=getType(col.type).formatter,fmtstr=col.fmtstr: fmtfunc(fmtstr, '' if v is None else v)
)
if trdict:
transformers[col].append(lambda v,trdict=trdict: v.translate(trdict))
options_safe_error = options.safe_error
for r in Progress(rows):
dispvals = []
for col, transforms in transformers.items():
try:
dispval = col.getValue(r)
except Exception as e:
exceptionCaught(e)
dispval = options_safe_error or str(e)
try:
for t in transforms:
if dispval is None:
dispval = ''
break
dispval = t(dispval)
except Exception as e:
dispval = str(dispval)
dispvals.append(dispval)
yield dispvals
@asyncthread
def save_tsv(p, vs):
'Write sheet to file `fn` as TSV.'
delim = options.get('delimiter', vs)
trdict = tsv_trdict(vs)
save_tsv_header(p, vs)
with p.open_text(mode='a') as fp:
for dispvals in genAllValues(vs.rows, vs.visibleCols, trdict, format=True):
fp.write(delim.join(dispvals))
fp.write('\n')
status('%s save finished' % p)
def append_tsv_row(vs, row):
'Append `row` to vs.source, creating file with correct headers if necessary. For internal use only.'
if not vs.source.exists():
with contextlib.suppress(FileExistsError):
parentdir = vs.source.parent.resolve()
if parentdir:
os.makedirs(parentdir)
save_tsv_header(vs.source, vs)
with vs.source.open_text(mode='a') as fp:
fp.write('\t'.join(col.getDisplayValue(row) for col in vs.visibleCols) + '\n')
visidata-1.5.2/visidata/loaders/sqlite.py 0000660 0001750 0001750 00000003537 13416500243 021127 0 ustar kefala kefala 0000000 0000000 from visidata import *
def open_sqlite(path):
vs = SqliteSheet(path.name + '_tables', path, 'sqlite_master')
vs.columns = vs.getColumns('sqlite_master')
vs.addCommand(ENTER, 'dive-row', 'vd.push(SqliteSheet(joinSheetnames(source.name, cursorRow[1]), sheet, cursorRow[1]))')
return vs
open_db = open_sqlite
class SqliteSheet(Sheet):
'Provide functionality for importing SQLite databases.'
def __init__(self, name, pathOrSheet, tableName):
super().__init__(name, source=pathOrSheet, tableName=tableName)
if isinstance(pathOrSheet, Sheet):
self.conn = pathOrSheet.conn
elif isinstance(pathOrSheet, Path):
import sqlite3
self.conn = sqlite3.connect(pathOrSheet.resolve())
# must not be @asyncthread due to sqlite lib being single-threaded
def reload(self):
tblname = self.tableName
self.columns = self.getColumns(tblname)
r = self.conn.execute('SELECT COUNT(*) FROM %s' % tblname).fetchall()
rowcount = r[0][0]
self.rows = []
for row in Progress(self.conn.execute("SELECT * FROM %s" % tblname), total=rowcount-1):
self.addRow(row)
def getColumns(self, tableName):
cols = []
for i, r in enumerate(self.conn.execute('PRAGMA TABLE_INFO(%s)' % tableName)):
c = ColumnItem(r[1], i)
t = r[2].lower()
if t == 'integer':
c.type = int
elif t == 'text':
c.type = str
elif t == 'blob':
c.type = str
elif t == 'real':
c.type = float
else:
status('unknown sqlite type "%s"' % t)
cols.append(c)
if r[-1]:
self.setKeys([c])
return cols
SqliteSheet.addCommand(ENTER, 'dive-row', 'error("sqlite dbs are readonly")')
visidata-1.5.2/visidata/loaders/zip.py 0000660 0001750 0001750 00000002135 13416252050 020421 0 ustar kefala kefala 0000000 0000000 import codecs
from visidata import *
class open_zip(Sheet):
'Provide wrapper around `zipfile` library for opening ZIP files.'
rowtype = 'files'
columns = [
ColumnAttr('filename'),
ColumnAttr('file_size', type=int),
Column('date_time', type=date, getter=lambda col,row: datetime.datetime(*row.date_time)),
ColumnAttr('compress_size', type=int)
]
def __init__(self, p):
super().__init__(p.name, source=p)
def reload(self):
import zipfile
with zipfile.ZipFile(self.source.resolve(), 'r') as zfp:
self.rows = zfp.infolist()
def openZipFileEntry(self, zi):
import zipfile
zfp = zipfile.ZipFile(self.source.resolve(), 'r')
decodedfp = codecs.iterdecode(zfp.open(zi), encoding=options.encoding, errors=options.encoding_errors)
return openSource(PathFd(zi.filename, decodedfp, filesize=zi.file_size))
open_zip.addCommand(ENTER, 'dive-row', 'vd.push(openZipFileEntry(cursorRow))')
open_zip.addCommand('g'+ENTER, 'dive-selected', 'for r in selectedRows or rows: vd.push(openZipFileEntry(r))')
visidata-1.5.2/visidata/loaders/fixed_width.py 0000660 0001750 0001750 00000003225 13416252050 022116 0 ustar kefala kefala 0000000 0000000
from visidata import *
option('fixed_rows', 1000, 'number of rows to check for fixed width columns')
def open_fixed(p):
return FixedWidthColumnsSheet(p.name, source=p)
class FixedWidthColumn(Column):
def __init__(self, name, i, j, **kwargs):
super().__init__(name, **kwargs)
self.i, self.j = i, j
def getValue(self, row):
return row[0][self.i:self.j]
def setValue(self, row, value):
value = str(value)[:self.j-self.i]
row[0] = row[0][:self.i] + '%-*s' % (self.j-self.i, value) + row[0][self.j:]
def columnize(rows):
'Generate (i,j) indexes for fixed-width columns found in rows'
## find all character columns that are not spaces
allNonspaces = set()
allNonspaces.add(max(len(r) for r in rows)+1)
for r in rows:
for i, ch in enumerate(r):
if not ch.isspace():
allNonspaces.add(i)
colstart = 0
prev = 0
# collapse fields
for i in allNonspaces:
if i > prev+1:
yield colstart, prev+1
colstart = i
prev = i
class FixedWidthColumnsSheet(Sheet):
rowtype = 'lines'
columns = [ColumnItem('line', 0)]
@asyncthread
def reload(self):
self.rows = []
for line in self.source:
self.addRow([line])
self.columns = []
# compute fixed width columns
for i, j in columnize(list(r[0] for r in self.rows[:options.fixed_rows])):
c = FixedWidthColumn('', i, j)
c.name = '_'.join(c.getValue(r) for r in self.rows[:options.header])
self.addColumn(c)
# discard header rows
self.rows = self.rows[options.header:]
visidata-1.5.2/visidata/urlcache.py 0000660 0001750 0001750 00000001473 13416252050 017760 0 ustar kefala kefala 0000000 0000000 import os
import os.path
import time
import urllib.request
import urllib.parse
from visidata import __version_info__, Path, options
def urlcache(url, cachesecs=24*60*60):
'Returns Path object to local cache of url contents.'
p = Path(os.path.join(options.visidata_dir, 'cache', urllib.parse.quote(url, safe='')))
if p.exists():
secs = time.time() - p.stat().st_mtime
if secs < cachesecs:
return p
if not p.parent.exists():
os.makedirs(p.parent.resolve(), exist_ok=True)
assert p.parent.is_dir(), p.parent
req = urllib.request.Request(url, headers={'User-Agent': __version_info__})
with urllib.request.urlopen(req) as fp:
ret = fp.read().decode('utf-8').strip()
with p.open_text(mode='w') as fpout:
fpout.write(ret)
return p
visidata-1.5.2/visidata/describe.py 0000660 0001750 0001750 00000006244 13416252050 017753 0 ustar kefala kefala 0000000 0000000 from statistics import mode, median, mean, stdev
from visidata import *
max_threads = 2
Sheet.addCommand('I', 'describe-sheet', 'vd.push(DescribeSheet(sheet.name+"_describe", source=[sheet]))')
globalCommand('gI', 'describe-all', 'vd.push(DescribeSheet("describe_all", source=vd.sheets))')
def isError(col, row):
try:
v = col.getValue(row)
if v is not None:
col.type(v)
return False
except Exception as e:
return True
class DescribeColumn(Column):
def __init__(self, name, **kwargs):
super().__init__(name, getter=lambda col,srccol: col.sheet.describeData[srccol].get(col.expr, ''), expr=name, **kwargs)
# rowdef: Column from source sheet
class DescribeSheet(ColumnsSheet):
# rowtype = 'columns'
columns = [
ColumnAttr('sheet', 'sheet'),
ColumnAttr('column', 'name'),
DescribeColumn('errors', type=len),
DescribeColumn('nulls', type=len),
DescribeColumn('distinct',type=len),
DescribeColumn('mode', type=str),
DescribeColumn('min', type=str),
DescribeColumn('max', type=str),
DescribeColumn('median', type=str),
DescribeColumn('mean', type=float),
DescribeColumn('stdev', type=float),
]
colorizers = [
RowColorizer(7, 'color_key_col', lambda s,c,r,v: r and r in r.sheet.keyCols),
]
@asyncthread
def reload(self):
super().reload()
self.rows = [c for c in self.rows if not c.hidden]
self.describeData = { col: {} for col in self.rows }
for srccol in Progress(self.rows, 'categorizing'):
if not srccol.hidden:
self.reloadColumn(srccol)
sync(max_threads)
@asyncthread
def reloadColumn(self, srccol):
d = self.describeData[srccol]
isNull = isNullFunc()
vals = list()
d['errors'] = list()
d['nulls'] = list()
d['distinct'] = set()
for sr in Progress(srccol.sheet.rows, 'calculating'):
try:
v = srccol.getValue(sr)
if isNull(v):
d['nulls'].append(sr)
else:
v = srccol.type(v)
vals.append(v)
d['distinct'].add(v)
except Exception as e:
d['errors'].append(sr)
d['mode'] = self.calcStatistic(d, mode, vals)
if isNumeric(srccol):
for func in [min, max, median, mean, stdev]:
d[func.__name__] = self.calcStatistic(d, func, vals)
def calcStatistic(self, d, func, *args, **kwargs):
r = wrapply(func, *args, **kwargs)
d[func.__name__] = r
return r
DescribeSheet.addCommand('zs', 'select-cell', 'cursorRow.sheet.select(cursorValue)')
DescribeSheet.addCommand('zu', 'unselect-cell', 'cursorRow.sheet.unselect(cursorValue)')
DescribeSheet.addCommand('z'+ENTER, 'dup-cell', 'isinstance(cursorValue, list) or error(cursorValue); vs=copy(cursorRow.sheet); vs.rows=cursorValue; vs.name+="_%s_%s"%(cursorRow.name,cursorCol.name); vd.push(vs)')
visidata-1.5.2/visidata/graph.py 0000660 0001750 0001750 00000013044 13416252050 017270 0 ustar kefala kefala 0000000 0000000 from visidata import *
option('color_graph_axis', 'bold', 'color for graph axis labels')
Sheet.addCommand('.', 'plot-column', 'vd.push(GraphSheet(sheet.name+"_graph", sheet, rows, keyCols, [cursorCol]))')
Sheet.addCommand('g.', 'plot-numerics', 'vd.push(GraphSheet(sheet.name+"_graph", sheet, rows, keyCols, numericCols(nonKeyVisibleCols)))')
def numericCols(cols):
# isNumeric from describe.py
return [c for c in cols if isNumeric(c)]
class InvertedCanvas(Canvas):
def zoomTo(self, bbox):
super().zoomTo(bbox)
self.fixPoint(Point(self.plotviewBox.xmin, self.plotviewBox.ymax), bbox.xymin)
def plotpixel(self, x, y, attr, row=None):
y = self.plotviewBox.ymax-y
self.pixels[y][x][attr].append(row)
def scaleY(self, canvasY):
'returns plotter y coordinate, with y-axis inverted'
plotterY = super().scaleY(canvasY)
return (self.plotviewBox.ymax-plotterY+4)
def canvasH(self, plotterY):
return (self.plotviewBox.ymax-plotterY)/self.yScaler
@property
def canvasMouse(self):
p = super().canvasMouse
p.y = self.visibleBox.ymin + (self.plotviewBox.ymax-self.plotterMouse.y)/self.yScaler
return p
# swap directions of up/down
InvertedCanvas.addCommand(None, 'go-up', 'sheet.cursorBox.ymin += cursorBox.h')
InvertedCanvas.addCommand(None, 'go-down', 'sheet.cursorBox.ymin -= cursorBox.h')
InvertedCanvas.addCommand(None, 'go-top', 'sheet.cursorBox.ymin = visibleBox.ymax')
InvertedCanvas.addCommand(None, 'go-bottom', 'sheet.cursorBox.ymin = visibleBox.ymin')
InvertedCanvas.addCommand(None, 'next-page', 't=(visibleBox.ymax-visibleBox.ymin); sheet.cursorBox.ymin -= t; sheet.visibleBox.ymin -= t; refresh()')
InvertedCanvas.addCommand(None, 'prev-page', 't=(visibleBox.ymax-visibleBox.ymin); sheet.cursorBox.ymin += t; sheet.visibleBox.ymin += t; refresh()')
InvertedCanvas.addCommand(None, 'go-down-small', 'sheet.cursorBox.ymin -= canvasCharHeight')
InvertedCanvas.addCommand(None, 'go-up-small', 'sheet.cursorBox.ymin += canvasCharHeight')
InvertedCanvas.addCommand(None, 'resize-cursor-shorter', 'sheet.cursorBox.h -= canvasCharHeight')
InvertedCanvas.addCommand(None, 'resize-cursor-taller', 'sheet.cursorBox.h += canvasCharHeight')
# provides axis labels, legend
class GraphSheet(InvertedCanvas):
def __init__(self, name, sheet, rows, xcols, ycols, **kwargs):
super().__init__(name, sheet, sourceRows=rows, **kwargs)
self.xcols = xcols
self.ycols = [ycol for ycol in ycols if isNumeric(ycol)] or fail('%s is non-numeric' % '/'.join(yc.name for yc in ycols))
@asyncthread
def reload(self):
nerrors = 0
nplotted = 0
self.reset()
status('loading data points')
catcols = [c for c in self.xcols if not isNumeric(c)]
numcols = numericCols(self.xcols)
for ycol in self.ycols:
for rownum, row in enumerate(Progress(self.sourceRows, 'plotting')): # rows being plotted from source
try:
k = tuple(c.getValue(row) for c in catcols) if catcols else (ycol.name,)
# convert deliberately to float (to e.g. linearize date)
graph_x = float(numcols[0].type(numcols[0].getValue(row))) if numcols else rownum
graph_y = ycol.type(ycol.getValue(row))
attr = self.plotColor(k)
self.point(graph_x, graph_y, attr, row)
nplotted += 1
except Exception:
nerrors += 1
if options.debug:
raise
status('loaded %d points (%d errors)' % (nplotted, nerrors))
self.setZoom(1.0)
self.refresh()
def setZoom(self, zoomlevel=None):
super().setZoom(zoomlevel)
self.createLabels()
def add_y_axis_label(self, frac):
amt = self.visibleBox.ymin + frac*self.visibleBox.h
srccol = self.ycols[0]
txt = srccol.format(srccol.type(amt))
# plot y-axis labels on the far left of the canvas, but within the plotview height-wise
attr = colors.color_graph_axis
self.plotlabel(0, self.plotviewBox.ymin + (1.0-frac)*self.plotviewBox.h, txt, attr)
def add_x_axis_label(self, frac):
amt = self.visibleBox.xmin + frac*self.visibleBox.w
txt = ','.join(xcol.format(xcol.type(amt)) for xcol in self.xcols if isNumeric(xcol))
# plot x-axis labels below the plotviewBox.ymax, but within the plotview width-wise
attr = colors.color_graph_axis
xmin = self.plotviewBox.xmin + frac*self.plotviewBox.w
if frac == 1.0:
# shift rightmost label to be readable
xmin -= max(len(txt)*2 - self.rightMarginPixels+1, 0)
self.plotlabel(xmin, self.plotviewBox.ymax+4, txt, attr)
def createLabels(self):
self.gridlabels = []
# y-axis
self.add_y_axis_label(1.00)
self.add_y_axis_label(0.75)
self.add_y_axis_label(0.50)
self.add_y_axis_label(0.25)
self.add_y_axis_label(0.00)
# x-axis
self.add_x_axis_label(1.00)
self.add_x_axis_label(0.75)
self.add_x_axis_label(0.50)
self.add_x_axis_label(0.25)
self.add_x_axis_label(0.00)
# TODO: if 0 line is within visible bounds, explicitly draw the axis
# TODO: grid lines corresponding to axis labels
xname = ','.join(xcol.name for xcol in self.xcols if isNumeric(xcol)) or 'row#'
self.plotlabel(0, self.plotviewBox.ymax+4, '%*s»' % (int(self.leftMarginPixels/2-2), xname), colors.color_graph_axis)
visidata-1.5.2/visidata/regex.py 0000660 0001750 0001750 00000004751 13416502136 017311 0 ustar kefala kefala 0000000 0000000 from visidata import *
Sheet.addCommand(':', 'split-col', 'addRegexColumns(makeRegexSplitter, sheet, cursorColIndex, cursorCol, cursorRow, input("split regex: ", type="regex-split"))')
Sheet.addCommand(';', 'capture-col', 'addRegexColumns(makeRegexMatcher, sheet, cursorColIndex, cursorCol, cursorRow, input("match regex: ", type="regex-capture"))')
Sheet.addCommand('*', 'addcol-subst', 'addColumn(Column(cursorCol.name + "_re", getter=regexTransform(cursorCol, input("transform column by regex: ", type="regex-subst"))), cursorColIndex+1)')
Sheet.addCommand('g*', 'setcol-subst', 'rex=input("transform column by regex: ", type="regex-subst"); setValuesFromRegex([cursorCol], selectedRows or rows, rex)')
Sheet.addCommand('gz*', 'setcol-subst-all', 'rex=input("transform column by regex: ", type="regex-subst"); setValuesFromRegex(visibleCols, selectedRows or rows, rex)')
replayableOption('regex_maxsplit', 0, 'maxsplit to pass to regex.split')
def makeRegexSplitter(regex, origcol):
return lambda row, regex=regex, origcol=origcol, maxsplit=options.regex_maxsplit: regex.split(origcol.getDisplayValue(row), maxsplit=maxsplit)
def makeRegexMatcher(regex, origcol):
return lambda row, regex=regex, origcol=origcol: regex.search(origcol.getDisplayValue(row)).groups()
def addRegexColumns(regexMaker, vs, colIndex, origcol, exampleRow, regexstr):
regex = re.compile(regexstr, regex_flags())
func = regexMaker(regex, origcol)
result = func(exampleRow)
for i, g in enumerate(result):
c = Column(origcol.name+'_re'+str(i), getter=lambda col,row,i=i,func=func: func(row)[i])
c.origCol = origcol
vs.addColumn(c, index=colIndex+i+1)
def regexTransform(origcol, instr):
i = indexWithEscape(instr, '/')
if i is None:
before = instr
after = ''
else:
before = instr[:i]
after = instr[i+1:]
return lambda col,row,origcol=origcol,before=before, after=after: re.sub(before, after, origcol.getDisplayValue(row), flags=regex_flags())
def indexWithEscape(s, char, escape_char='\\'):
i=0
while i < len(s):
if s[i] == escape_char:
i += 1
elif s[i] == char:
return i
i += 1
return None
@asyncthread
def setValuesFromRegex(cols, rows, rex):
transforms = [regexTransform(col, rex) for col in cols]
for r in Progress(rows, 'replacing'):
for col, transform in zip(cols, transforms):
col.setValueSafe(r, transform(col, r))
for col in cols:
col.recalc()
visidata-1.5.2/visidata/namedlist.py 0000660 0001750 0001750 00000001421 13416500243 020143 0 ustar kefala kefala 0000000 0000000 import operator
def itemsetter(i):
def g(obj, v):
obj[i] = v
return g
def namedlist(objname, fieldnames):
'like namedtuple but editable'
class NamedListTemplate(list):
__name__ = objname
_fields = fieldnames
def __init__(self, L=None, **kwargs):
if L is None:
L = [None]*len(fieldnames)
super().__init__(L)
for k, v in kwargs.items():
setattr(self, k, v)
@classmethod
def length(cls):
return len(cls._fields)
for i, attrname in enumerate(fieldnames):
# create property getter/setter for each field
setattr(NamedListTemplate, attrname, property(operator.itemgetter(i), itemsetter(i)))
return NamedListTemplate
visidata-1.5.2/visidata/pyobj.py 0000660 0001750 0001750 00000021561 13416500243 017315 0 ustar kefala kefala 0000000 0000000 from visidata import *
option('visibility', 0, 'visibility level')
globalCommand('^X', 'pyobj-expr', 'expr = input("eval: ", "expr", completer=CompleteExpr()); push_pyobj(expr, evalexpr(expr, None))')
globalCommand('g^X', 'exec-python', 'expr = input("exec: ", "expr", completer=CompleteExpr()); exec(expr, getGlobals())')
globalCommand('z^X', 'pyobj-expr-row', 'expr = input("eval over current row: ", "expr", completer=CompleteExpr()); push_pyobj(expr, evalexpr(expr, cursorRow))')
Sheet.addCommand('^Y', 'pyobj-row', 'status(type(cursorRow)); push_pyobj("%s[%s]" % (sheet.name, cursorRowIndex), cursorRow)')
Sheet.addCommand('z^Y', 'pyobj-cell', 'status(type(cursorValue)); push_pyobj("%s[%s].%s" % (sheet.name, cursorRowIndex, cursorCol.name), cursorValue)')
globalCommand('g^Y', 'pyobj-sheet', 'status(type(sheet)); push_pyobj(sheet.name+"_sheet", sheet)')
Sheet.addCommand('(', 'expand-col', 'expand_cols_deep(sheet, [cursorCol], cursorRow, depth=0)')
Sheet.addCommand('g(', 'expand-cols', 'expand_cols_deep(sheet, visibleCols, cursorRow, depth=0)')
Sheet.addCommand('z(', 'expand-col-depth', 'expand_cols_deep(sheet, [cursorCol], cursorRow, depth=int(input("expand depth=", value=1)))')
Sheet.addCommand('gz(', 'expand-cols-depth', 'expand_cols_deep(sheet, visibleCols, cursorRow, depth=int(input("expand depth=", value=1)))')
Sheet.addCommand(')', 'contract-col', 'closeColumn(sheet, cursorCol)')
class PythonSheet(Sheet):
pass
# used as ENTER in several pyobj sheets
PythonSheet.addCommand(None, 'dive-row', 'push_pyobj("%s[%s]" % (name, cursorRowIndex), cursorRow)')
bindkey(ENTER, 'dive-row')
def expand_cols_deep(sheet, cols, row, depth=0): # depth == 0 means drill all the way
'expand all visible columns of containers to the given depth (0=fully)'
ret = []
for col in cols:
newcols = _addExpandedColumns(col, row, sheet.columns.index(col))
if depth != 1: # countdown not yet complete, or negative (indefinite)
ret.extend(expand_cols_deep(sheet, newcols, row, depth-1))
return ret
def _addExpandedColumns(col, row, idx):
val = col.getTypedValueNoExceptions(row)
if isinstance(val, dict):
ret = [
ExpandedColumn('%s.%s' % (col.name, k), type=deduceType(val[k]), origCol=col, key=k)
for k in val
]
elif isinstance(val, (list, tuple)):
ret = [
ExpandedColumn('%s[%s]' % (col.name, k), type=deduceType(val[k]), origCol=col, key=k)
for k in range(len(val))
]
else:
return []
for i, c in enumerate(ret):
col.sheet.addColumn(c, idx+i+1)
col.hide()
return ret
def deduceType(v):
if isinstance(v, (float, int)):
return type(v)
else:
return anytype
class ExpandedColumn(Column):
def calcValue(self, row):
return getitemdef(self.origCol.getValue(row), self.key)
def setValue(self, row, value):
self.origCol.getValue(row)[self.key] = value
def closeColumn(sheet, col):
col.origCol.width = options.default_width
cols = [c for c in sheet.columns if getattr(c, "origCol", None) is not getattr(col, "origCol", col)]
sheet.columns = cols
#### generic list/dict/object browsing
def push_pyobj(name, pyobj):
vs = load_pyobj(name, pyobj)
if vs:
return vd().push(vs)
else:
error("cannot push '%s' as pyobj" % type(pyobj).__name__)
def view(obj):
run(load_pyobj(getattr(obj, '__name__', ''), obj))
def load_pyobj(name, pyobj):
'Return Sheet object of appropriate type for given sources in `args`.'
if isinstance(pyobj, list) or isinstance(pyobj, tuple):
if getattr(pyobj, '_fields', None): # list of namedtuple
return SheetNamedTuple(name, pyobj)
else:
return SheetList(name, pyobj)
elif isinstance(pyobj, dict):
return SheetDict(name, pyobj)
elif isinstance(pyobj, object):
return SheetObject(name, pyobj)
else:
error("cannot load '%s' as pyobj" % type(pyobj).__name__)
def open_pyobj(path):
'Provide wrapper for `load_pyobj`.'
return load_pyobj(path.name, eval(path.read_text()))
def getPublicAttrs(obj):
'Return all public attributes (not methods or `_`-prefixed) on object.'
return [k for k in dir(obj) if not k.startswith('_') and not callable(getattr(obj, k))]
def PyobjColumns(obj):
'Return columns for each public attribute on an object.'
return [ColumnAttr(k, type(getattr(obj, k))) for k in getPublicAttrs(obj)]
def AttrColumns(attrnames):
'Return column names for all elements of list `attrnames`.'
return [ColumnAttr(name) for name in attrnames]
def DictKeyColumns(d):
'Return a list of Column objects from dictionary keys.'
return [ColumnItem(k, k, type=deduceType(d[k])) for k in d.keys()]
def SheetList(name, src, **kwargs):
'Creates a Sheet from a list of homogenous dicts or namedtuples.'
if not src:
status('no content in ' + name)
return
if isinstance(src[0], dict):
return ListOfDictSheet(name, source=src, **kwargs)
elif isinstance(src[0], tuple):
if getattr(src[0], '_fields', None): # looks like a namedtuple
return ListOfNamedTupleSheet(name, source=src, **kwargs)
# simple list
return ListOfPyobjSheet(name, source=src, **kwargs)
class ListOfPyobjSheet(PythonSheet):
rowtype = 'python objects'
def reload(self):
self.rows = self.source
self.columns = [Column(self.name,
getter=lambda col,row: row,
setter=lambda col,row,val: setitem(col.sheet.source, col.sheet.source.index(row), val))]
# rowdef: dict
class ListOfDictSheet(PythonSheet):
rowtype = 'dicts'
def reload(self):
self.columns = DictKeyColumns(self.source[0])
self.rows = self.source
# rowdef: namedtuple
class ListOfNamedTupleSheet(PythonSheet):
rowtype = 'namedtuples'
def reload(self):
self.columns = [ColumnItem(k, i) for i, k in enumerate(self.source[0]._fields)]
self.rows = self.source
# rowdef: PyObj
class SheetNamedTuple(PythonSheet):
'a single namedtuple, with key and value columns'
rowtype = 'values'
columns = [ColumnItem('name', 0), ColumnItem('value', 1)]
def __init__(self, name, src, **kwargs):
super().__init__(name, source=src, **kwargs)
def reload(self):
self.rows = list(zip(self.source._fields, self.source))
def dive(self):
push_pyobj(joinSheetnames(self.name, self.cursorRow[0]), self.cursorRow[1])
SheetNamedTuple.addCommand(ENTER, 'dive-row', 'dive()')
class SheetDict(PythonSheet):
rowtype = 'items' # rowdef: [key, value] (editable)
def __init__(self, name, source, **kwargs): # source is dict()
super().__init__(name, source=source, **kwargs)
def reload(self):
self.columns = [ColumnItem('key', 0)]
self.rows = list(list(x) for x in self.source.items())
self.columns.append(ColumnItem('value', 1))
def edit(self):
self.source[self.cursorRow[0]][1] = self.editCell(1)
self.cursorRowIndex += 1
self.reload()
def dive(self):
push_pyobj(joinSheetnames(self.name, self.cursorRow[0]), self.cursorRow[1])
SheetDict.addCommand('e', 'edit-cell', 'edit()')
SheetDict.addCommand(ENTER, 'dive-row', 'dive()')
class ColumnSourceAttr(Column):
'Use row as attribute name on sheet source'
def calcValue(self, attrname):
return getattr(self.sheet.source, attrname)
def setValue(self, attrname, value):
return setattr(self.sheet.source, attrname, value)
# rowdef: attrname
class SheetObject(PythonSheet):
rowtype = 'attributes'
def __init__(self, name, obj, **kwargs):
super().__init__(name, source=obj, **kwargs)
def reload(self):
self.rows = []
vislevel = options.visibility
for r in dir(self.source):
try:
if vislevel <= 1 and r.startswith('_'): continue
if vislevel <= 0 and callable(getattr(self.source, r)): continue
self.addRow(r)
except Exception:
pass
self.columns = [
Column(type(self.source).__name__ + '_attr'),
ColumnSourceAttr('value'),
ColumnExpr('docstring', 'value.__doc__ if callable(value) else type(value).__name__')
]
self.recalc()
self.setKeys(self.columns[0:1])
SheetObject.addCommand(ENTER, 'dive-row', 'v = getattr(source, cursorRow); push_pyobj(joinSheetnames(name, cursorRow), v() if callable(v) else v)')
SheetObject.addCommand('e', 'edit-cell', 'setattr(source, cursorRow, type(getattr(source, cursorRow))(editCell(1))); sheet.cursorRowIndex += 1; reload()')
SheetObject.addCommand('v', 'visibility', 'options.set("visibility", 0 if options.visibility else 2, sheet); reload()')
SheetObject.addCommand('gv', 'show-hidden', 'options.visibility = 2; reload()')
SheetObject.addCommand('zv', 'hide-hidden', 'options.visibility -= 1; reload()')
visidata-1.5.2/visidata/clipboard.py 0000660 0001750 0001750 00000012330 13416500243 020123 0 ustar kefala kefala 0000000 0000000 from copy import copy
import shutil
import subprocess
import sys
import tempfile
import functools
from visidata import vd, asyncthread, sync, status, fail, option, options
from visidata import Sheet, saveSheets
vd.cliprows = [] # list of (source_sheet, source_row_idx, source_row)
vd.clipcells = [] # list of strings
Sheet.addCommand('y', 'copy-row', 'vd.cliprows = [(sheet, cursorRowIndex, cursorRow)]')
Sheet.addCommand('d', 'delete-row', 'vd.cliprows = [(sheet, cursorRowIndex, rows.pop(cursorRowIndex))]')
Sheet.addCommand('p', 'paste-after', 'rows[cursorRowIndex+1:cursorRowIndex+1] = list(deepcopy(r) for s,i,r in vd.cliprows)')
Sheet.addCommand('P', 'paste-before', 'rows[cursorRowIndex:cursorRowIndex] = list(deepcopy(r) for s,i,r in vd.cliprows)')
Sheet.addCommand('gd', 'delete-selected', 'vd.cliprows = list((None, i, r) for i, r in enumerate(selectedRows)); deleteSelected()')
Sheet.addCommand('gy', 'copy-selected', 'vd.cliprows = list((None, i, r) for i, r in enumerate(selectedRows)); status("%d %s to clipboard" % (len(vd.cliprows), rowtype))')
Sheet.addCommand('zy', 'copy-cell', 'vd.clipcells = [cursorDisplay]')
Sheet.addCommand('zp', 'paste-cell', 'cursorCol.setValuesTyped([cursorRow], vd.clipcells[0])')
Sheet.addCommand('zd', 'delete-cell', 'vd.clipcells = [cursorDisplay]; cursorCol.setValues([cursorRow], None)')
Sheet.addCommand('gzd', 'delete-cells', 'vd.clipcells = list(sheet.cursorCol.getDisplayValue(r) for r in selectedRows); cursorCol.setValues(selectedRows, None)')
Sheet.addCommand('gzy', 'copy-cells', 'vd.clipcells = [sheet.cursorCol.getDisplayValue(r) for r in selectedRows]; status("%d values to clipboard" % len(vd.clipcells))')
Sheet.addCommand('gzp', 'paste-cells', 'for r, v in zip(selectedRows or rows, itertools.cycle(vd.clipcells)): cursorCol.setValuesTyped([r], v)')
Sheet.addCommand('Y', 'syscopy-row', 'saveToClipboard(sheet, [cursorRow], input("copy current row to system clipboard as filetype: ", value=options.save_filetype))')
Sheet.addCommand('gY', 'syscopy-selected', 'saveToClipboard(sheet, selectedRows or rows, input("copy rows to system clipboard as filetype: ", value=options.save_filetype))')
Sheet.addCommand('zY', 'syscopy-cell', 'copyToClipboard(cursorDisplay)')
Sheet.addCommand('gzY', 'syscopy-cells', 'copyToClipboard("\\n".join(sheet.cursorCol.getDisplayValue(r) for r in selectedRows))')
Sheet.bindkey('KEY_DC', 'delete-cell'),
Sheet.bindkey('gKEY_DC', 'delete-cells'),
option('clipboard_copy_cmd', '', 'command to copy stdin to system clipboard')
__clipboard_commands = [
('win32', 'clip', ''), # Windows Vista+
('darwin', 'pbcopy', 'w'), # macOS
(None, 'xclip', '-selection clipboard -filter'), # Linux etc.
(None, 'xsel', '--clipboard --input'), # Linux etc.
]
def detect_command(cmdlist):
'''Detect available clipboard util and return cmdline to copy data to the system clipboard.
cmddict is list of (platform, progname, argstr).'''
for platform, command, args in cmdlist:
if platform is None or sys.platform == platform:
path = shutil.which(command)
if path:
return ' '.join([path, args])
return ''
detect_clipboard_command = lambda: detect_command(__clipboard_commands)
@functools.lru_cache()
def clipboard():
'Detect cmd and set option at first use, to allow option to be changed by user later.'
if not options.clipboard_copy_cmd:
options.clipboard_copy_cmd = detect_clipboard_command()
return _Clipboard()
class _Clipboard:
'Cross-platform helper to copy a cell or multiple rows to the system clipboard.'
@property
def command(self):
'Return cmdline cmd+args (as list for Popen) to copy data to the system clipboard.'
cmd = options.clipboard_copy_cmd or fail('options.clipboard_copy_cmd not set')
return cmd.split()
def copy(self, value):
'Copy a cell to the system clipboard.'
with tempfile.NamedTemporaryFile() as temp:
with open(temp.name, 'w', encoding=options.encoding) as fp:
fp.write(str(value))
p = subprocess.Popen(
self.command,
stdin=open(temp.name, 'r', encoding=options.encoding),
stdout=subprocess.DEVNULL)
p.communicate()
def save(self, vs, filetype):
'Copy rows to the system clipboard.'
# use NTF to generate filename and delete file on context exit
with tempfile.NamedTemporaryFile(suffix='.'+filetype) as temp:
saveSheets(temp.name, vs)
sync(1)
p = subprocess.Popen(
self.command,
stdin=open(temp.name, 'r', encoding=options.encoding),
stdout=subprocess.DEVNULL,
close_fds=True)
p.communicate()
def copyToClipboard(value):
'copy single value to system clipboard'
clipboard().copy(value)
status('copied value to clipboard')
@asyncthread
def saveToClipboard(sheet, rows, filetype=None):
'copy rows from sheet to system clipboard'
filetype = filetype or options.save_filetype
vs = copy(sheet)
vs.rows = rows
status('copying rows to clipboard')
clipboard().save(vs, filetype)
visidata-1.5.2/visidata/tidydata.py 0000660 0001750 0001750 00000006162 13416252050 017775 0 ustar kefala kefala 0000000 0000000 from visidata import *
Sheet.addCommand('M', 'melt', 'vd.push(MeltedSheet(sheet))')
Sheet.addCommand('gM', 'melt-regex', 'vd.push(MeltedSheet(sheet, regex=input("regex to split colname: ", value="(.*)_(.*)", type="regex-capture")))')
melt_var_colname = 'Variable' # column name to use for the melted variable name
melt_value_colname = 'Value' # column name to use for the melted value
# rowdef: {0:sourceRow, 1:Category1, ..., N:CategoryN, ColumnName:Column, ...}
class MeltedSheet(Sheet):
"Perform 'melt', the inverse of 'pivot', on input sheet."
rowtype = 'melted values'
def __init__(self, sheet, regex='(.*)', **kwargs):
super().__init__(sheet.name + '_melted', source=sheet, **kwargs)
self.regex = regex
@asyncthread
def reload(self):
isNull = isNullFunc()
sheet = self.source
self.columns = [SubrowColumn(c.name, c, 0) for c in sheet.keyCols]
self.setKeys(self.columns)
colsToMelt = [copy(c) for c in sheet.nonKeyVisibleCols]
# break down Category1_Category2_ColumnName as per regex
valcols = collections.OrderedDict() # ('Category1', 'Category2') -> list of tuple('ColumnName', Column)
for c in colsToMelt:
c.aggregators = [aggregators['max']]
m = re.match(self.regex, c.name)
if m:
if len(m.groups()) == 1:
varvals = m.groups()
valcolname = melt_value_colname
else:
*varvals, valcolname = m.groups()
cats = tuple(varvals)
if cats not in valcols:
valcols[cats] = []
valcols[cats].append((valcolname, c))
ncats = len(varvals)
else:
status('"%s" column does not match regex, skipping' % c.name)
othercols = set()
for colnames, cols in valcols.items():
for cname, _ in cols:
othercols.add(cname)
if ncats == 1:
self.columns.append(ColumnItem(melt_var_colname, 1))
else:
for i in range(ncats):
self.columns.append(ColumnItem('%s%d' % (melt_var_colname, i+1), i+1))
for cname in othercols:
self.columns.append(Column(cname,
getter=lambda col,row,cname=cname: row[cname].getValue(row[0]),
setter=lambda col,row,val,cname=cname: row[cname].setValue(row[0], val),
aggregators=[aggregators['max']]))
self.rows = []
for r in Progress(self.source.rows, 'melting'):
for colnames, cols in valcols.items():
meltedrow = {}
for varval, c in cols:
try:
if not isNull(c.getValue(r)):
meltedrow[varval] = c
except Exception as e:
pass
if meltedrow: # remove rows with no content (all nulls)
meltedrow[0] = r
for i, colname in enumerate(colnames):
meltedrow[i+1] = colname
self.addRow(meltedrow)
visidata-1.5.2/visidata/_types.py 0000660 0001750 0001750 00000005547 13416252050 017503 0 ustar kefala kefala 0000000 0000000 # VisiData uses Python native int, float, str, and adds simple date, currency, and anytype.
import functools
import datetime
from visidata import options, theme, Sheet, TypedWrapper
from .vdtui import vdtype
try:
import dateutil.parser
except ImportError:
pass
theme('disp_date_fmt','%Y-%m-%d', 'default fmtstr to strftime for date values')
Sheet.addCommand('z~', 'type-any', 'cursorCol.type = anytype'),
Sheet.addCommand('~', 'type-string', 'cursorCol.type = str'),
Sheet.addCommand('@', 'type-date', 'cursorCol.type = date'),
Sheet.addCommand('#', 'type-int', 'cursorCol.type = int'),
Sheet.addCommand('z#', 'type-len', 'cursorCol.type = len'),
Sheet.addCommand('$', 'type-currency', 'cursorCol.type = currency'),
Sheet.addCommand('%', 'type-float', 'cursorCol.type = float'),
floatchars='+-0123456789.'
def currency(s=''):
'dirty float (strip non-numeric characters)'
if isinstance(s, str):
s = ''.join(ch for ch in s if ch in floatchars)
return float(s) if s else TypedWrapper(float, None)
class date(datetime.datetime):
'datetime wrapper, constructed from time_t or from str with dateutil.parse'
def __new__(cls, s=None):
'datetime is immutable so needs __new__ instead of __init__'
if s is None:
r = datetime.datetime.now()
elif isinstance(s, int) or isinstance(s, float):
r = datetime.datetime.fromtimestamp(s)
elif isinstance(s, str):
r = dateutil.parser.parse(s)
elif isinstance(s, datetime.datetime):
r = s
else:
raise Exception('invalid type for date %s' % type(s).__name__)
t = r.timetuple()
return super().__new__(cls, *t[:6], microsecond=r.microsecond, tzinfo=r.tzinfo)
def __str__(self):
return self.strftime(options.disp_date_fmt)
def __float__(self):
return self.timestamp()
def __radd__(self, n):
return self.__add__(n)
def __add__(self, n):
'add n days (int or float) to the date'
if isinstance(n, (int, float)):
n = datetime.timedelta(days=n)
return date(super().__add__(n))
def __sub__(self, n):
'subtract n days (int or float) from the date. or subtract another date for a timedelta'
if isinstance(n, (int, float)):
n = datetime.timedelta(days=n)
elif isinstance(n, (date, datetime.datetime)):
return datedelta(super().__sub__(n).total_seconds()/(24*60*60))
return super().__sub__(n)
class datedelta(datetime.timedelta):
def __float__(self):
return self.total_seconds()
vdtype(date, '@', '', formatter=lambda fmtstr,val: val.strftime(fmtstr or options.disp_date_fmt))
vdtype(currency, '$', '{:,.02f}')
# simple constants, for expressions like 'timestamp+15*minutes'
years=365.25
months=30.0
weeks=7.0
days=1.0
hours=days/24
minutes=days/(24*60)
seconds=days/(24*60*60)
visidata-1.5.2/visidata/path.py 0000660 0001750 0001750 00000013101 13416500243 017115 0 ustar kefala kefala 0000000 0000000 import os
import os.path
import sys
from .vdtui import *
replayableOption('skip', 0, 'skip first N lines of text input')
class Path:
'File and path-handling class, modeled on `pathlib.Path`.'
def __init__(self, fqpn):
self.fqpn = fqpn
fn = self.parts[-1]
self.name, self.ext = os.path.splitext(fn)
# check if file is compressed
if self.ext in ['.gz', '.bz2', '.xz']:
self.compression = self.ext[1:]
self.name, self.ext = os.path.splitext(self.name)
else:
self.compression = None
self.suffix = self.ext[1:]
self._stat = None
def open_text(self, mode='rt'):
if 't' not in mode:
mode += 't'
if self.fqpn == '-':
if 'r' in mode:
return sys.stdin
elif 'w' in mode:
return sys.stdout
else:
error('invalid mode "%s" for Path.open_text()' % mode)
return sys.stderr
return self._open(self.resolve(), mode=mode, encoding=options.encoding, errors=options.encoding_errors)
def _open(self, *args, **kwargs):
if self.compression == 'gz':
import gzip
return gzip.open(*args, **kwargs)
elif self.compression == 'bz2':
import bz2
return bz2.open(*args, **kwargs)
elif self.compression == 'xz':
import lzma
return lzma.open(*args, **kwargs)
else:
return open(*args, **kwargs)
def __iter__(self):
skip = options.skip
with Progress(total=self.filesize) as prog:
for i, line in enumerate(self.open_text()):
prog.addProgress(len(line))
if i < skip:
continue
yield line[:-1]
def read_text(self):
with self.open_text() as fp:
return fp.read()
def open_bytes(self):
return self._open(self.resolve(), 'rb')
def read_bytes(self):
with self._open(self.resolve(), 'rb') as fp:
return fp.read()
def is_dir(self):
return os.path.isdir(self.resolve())
def exists(self):
return os.path.exists(self.resolve())
def iterdir(self):
return [self.parent] + [Path(os.path.join(self.fqpn, f)) for f in os.listdir(self.resolve())]
def stat(self, force=False):
if force or self._stat is None:
try:
self._stat = os.stat(self.resolve())
except Exception as e:
self._stat = e
return self._stat
def resolve(self):
'Resolve pathname shell variables and ~userdir'
return os.path.expandvars(os.path.expanduser(self.fqpn))
def relpath(self, start):
ourpath = self.resolve()
if ourpath == start:
return ''
return os.path.relpath(os.path.realpath(ourpath), start)
def with_name(self, name):
args = list(self.parts[:-1]) + [name]
fn = os.path.join(*args)
return Path(fn)
def joinpath(self, *other):
args = list(self.parts) + list(other)
fn = os.path.join(*args)
return Path(fn)
@property
def parts(self):
'Return list of path parts'
return os.path.split(self.fqpn)
@property
def parent(self):
'Return Path to parent directory.'
return Path(os.path.join(*self.parts[:-1]))
@property
def filesize(self):
return self.stat().st_size
def __str__(self):
return self.fqpn
def __lt__(self, a):
return self.name < a.name
class UrlPath(Path):
def __init__(self, url):
from urllib.parse import urlparse
self.url = url
self.obj = urlparse(url)
super().__init__(self.obj.path)
def __str__(self):
return self.url
def __getattr__(self, k):
return getattr(self.obj, k)
class PathFd(Path):
'minimal Path interface to satisfy a tsv loader'
def __init__(self, pathname, fp, filesize=0):
super().__init__(pathname)
self.fp = fp
self.alreadyRead = [] # shared among all RepeatFile instances
self._filesize = filesize
def read_text(self):
return self.fp.read()
def open_text(self):
return RepeatFile(self)
@property
def filesize(self):
return self._filesize
class RepeatFile:
def __init__(self, pathfd):
self.pathfd = pathfd
self.iter = RepeatFileIter(self)
def __enter__(self):
self.iter = RepeatFileIter(self)
return self
def __exit__(self, a,b,c):
pass
def read(self, n=None):
r = ''
if n is None:
n = 10**12 # some too huge number
while len(r) < n:
try:
s = next(self.iter)
r += s + '\n'
n += len(r)
except StopIteration:
break # end of file
return r
def seek(self, n):
assert n == 0, 'RepeatFile can only seek to beginning'
self.iter = RepeatFileIter(self)
def __iter__(self):
return RepeatFileIter(self)
def __next__(self):
return next(self.iter)
class RepeatFileIter:
def __init__(self, rf):
self.rf = rf
self.nextIndex = 0
def __iter__(self):
return RepeatFileIter(self.rf)
def __next__(self):
if self.nextIndex < len(self.rf.pathfd.alreadyRead):
r = self.rf.pathfd.alreadyRead[self.nextIndex]
else:
r = next(self.rf.pathfd.fp)
self.rf.pathfd.alreadyRead.append(r)
self.nextIndex += 1
return r
visidata-1.5.2/visidata/freeze.py 0000660 0001750 0001750 00000003514 13416500243 017450 0 ustar kefala kefala 0000000 0000000 from visidata import *
from copy import deepcopy
Sheet.addCommand("'", 'freeze-col', 'addColumn(StaticColumn(sheet.rows, cursorCol), cursorColIndex+1)')
Sheet.addCommand("g'", 'freeze-sheet', 'vd.push(StaticSheet(sheet)); status("pushed frozen copy of "+name)')
Sheet.addCommand("z'", 'cache-col', 'cursorCol.resetCache()')
Sheet.addCommand("gz'", 'cache-cols', 'for c in visibleCols: c.resetCache()')
def resetCache(self):
self._cachedValues = collections.OrderedDict()
status("reset cache for " + self.name)
Column.resetCache = resetCache
def StaticColumn(rows, col):
frozencol = SettableColumn(col.name+'_frozen', width=col.width, type=col.type, fmtstr=col.fmtstr)
@asyncthread
def calcRows_async(frozencol, rows, col):
for r in Progress(rows, 'calculating'):
try:
frozencol.setValue(r, col.getTypedValue(r))
except Exception as e:
frozencol.setValue(r, e)
calcRows_async(frozencol, rows, col)
return frozencol
class StaticSheet(Sheet):
'A copy of the source sheet with all cells frozen.'
def __init__(self, source):
super().__init__(source.name + "'", source=source)
self.columns = []
for i, col in enumerate(self.source.columns):
colcopy = ColumnItem(col.name, i, width=col.width, type=col.type, fmtstr=col.fmtstr)
self.addColumn(colcopy)
if col in self.source.keyCols:
self.setKeys([colcopy])
@asyncthread
def reload(self):
self.rows = []
for r in Progress(self.source.rows, 'calculating'):
row = []
self.rows.append(row)
for col in self.source.columns:
try:
row.append(col.getTypedValueOrException(r))
except Exception as e:
row.append(None)
visidata-1.5.2/visidata/vdtui.py 0000770 0001750 0001750 00000313446 13416503051 017335 0 ustar kefala kefala 0000000 0000000 #!/usr/bin/env python3
#
# Copyright 2017-2018 Saul Pwanson http://saul.pw/vdtui
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
'vdtui: a curses framework for columnar data'
# Just include this whole file in your project as-is. If you do make
# modifications, please keep the base vdtui version and append your own id and
# version.
__version__ = '1.5.2'
__version_info__ = 'saul.pw/vdtui v' + __version__
__author__ = 'Saul Pwanson '
__license__ = 'MIT'
__status__ = 'Production/Stable'
__copyright__ = 'Copyright (c) 2016-2018 ' + __author__
from builtins import *
import sys
import os
import collections
from collections import defaultdict
from copy import copy, deepcopy
from contextlib import suppress
import curses
import datetime
import functools
import io
import itertools
import string
import re
import textwrap
import threading
import traceback
import time
import inspect
import weakref
class EscapeException(BaseException):
'Inherits from BaseException to avoid "except Exception" clauses. Do not use a blanket "except:" or the task will be uncancelable.'
pass
class ExpectedException(Exception):
'an expected exception'
pass
vd = None # will be filled in later
# [settingname] -> { objname(Sheet-instance/Sheet-type/'override'/'global'): Option/Command/longname }
class SettingsMgr(collections.OrderedDict):
def __init__(self):
super().__init__()
self.allobjs = {}
def objname(self, obj):
if isinstance(obj, str):
v = obj
elif obj is None:
v = 'override'
elif isinstance(obj, BaseSheet):
v = obj.name
elif issubclass(obj, BaseSheet):
v = obj.__name__
else:
return None
self.allobjs[v] = obj
return v
def getobj(self, objname):
'Inverse of objname(obj); returns obj if available'
return self.allobjs.get(objname)
def set(self, k, v, obj='override'):
'obj is a Sheet instance, or a Sheet [sub]class. obj="override" means override all; obj="default" means last resort.'
if k not in self:
self[k] = dict()
self[k][self.objname(obj)] = v
return v
def setdefault(self, k, v):
return self.set(k, v, 'global')
def _mappings(self, obj):
if obj:
mappings = [self.objname(obj)]
mro = [self.objname(cls) for cls in inspect.getmro(type(obj))]
mappings.extend(mro)
else:
mappings = []
mappings += ['override', 'global']
return mappings
def _get(self, key, obj=None):
d = self.get(key, None)
if d:
for m in self._mappings(obj or vd.sheet):
v = d.get(m)
if v:
return v
def iter(self, obj=None):
'Iterate through all keys considering context of obj. If obj is None, uses the context of the top sheet.'
if obj is None and vd:
obj = vd.sheet
for o in self._mappings(obj):
for k in self.keys():
for o2 in self[k]:
if o == o2:
yield (k, o), self[k][o2]
class Command:
def __init__(self, longname, execstr):
self.longname = longname
self.execstr = execstr
self.helpstr = ''
def globalCommand(keystrokes, longname, execstr, helpstr=''):
commands.setdefault(longname, Command(longname, execstr))
if keystrokes:
assert not bindkeys._get(keystrokes), keystrokes
bindkeys.setdefault(keystrokes, longname)
def bindkey(keystrokes, longname):
bindkeys.setdefault(keystrokes, longname)
def bindkey_override(keystrokes, longname):
bindkeys.set(keystrokes, longname)
class Option:
def __init__(self, name, value, helpstr=''):
self.name = name
self.value = value
self.helpstr = helpstr
self.replayable = False
def __str__(self):
return str(self.value)
class OptionsObject:
'minimalist options framework'
def __init__(self, mgr):
object.__setattr__(self, '_opts', mgr)
object.__setattr__(self, '_cache', {})
def keys(self, obj=None):
for k, d in self._opts.items():
if obj is None or self._opts.objname(obj) in d:
yield k
def _get(self, k, obj=None):
'Return Option object for k in context of obj. Cache result until any set().'
opt = self._cache.get((k, obj), None)
if opt is None:
opt = self._opts._get(k, obj)
self._cache[(k, obj or vd.sheet)] = opt
return opt
def _set(self, k, v, obj=None, helpstr=''):
self._cache.clear() # invalidate entire cache on any set()
return self._opts.set(k, Option(k, v, helpstr), obj)
def get(self, k, obj=None):
return self._get(k, obj).value
def getdefault(self, k):
return self._get(k, 'global').value
def set(self, k, v, obj='override'):
opt = self._get(k)
if opt:
curval = opt.value
t = type(curval)
if v is None and curval is not None:
v = t() # empty value
elif isinstance(v, str) and t is bool: # special case for bool options
v = v and (v[0] not in "0fFnN") # ''/0/false/no are false, everything else is true
elif type(v) is t: # if right type, no conversion
pass
elif curval is None: # if None, do not apply type conversion
pass
else:
v = t(v)
if curval != v and self._get(k, 'global').replayable:
vd().callHook('set_option', k, v, obj)
else:
curval = None
warning('setting unknown option %s' % k)
return self._set(k, v, obj)
def setdefault(self, k, v, helpstr):
return self._set(k, v, 'global', helpstr=helpstr)
def getall(self, kmatch):
return {obj:opt for (k, obj), opt in self._opts.items() if k == kmatch}
def __getattr__(self, k): # options.foo
return self.__getitem__(k)
def __setattr__(self, k, v): # options.foo = v
self.__setitem__(k, v)
def __getitem__(self, k): # options[k]
opt = self._get(k)
if not opt:
error('no option "%s"' % k)
return opt.value
def __setitem__(self, k, v): # options[k] = v
self.set(k, v)
def __call__(self, prefix=''):
return { optname[len(prefix):] : options[optname]
for optname in options.keys()
if optname.startswith(prefix) }
commands = SettingsMgr()
bindkeys = SettingsMgr()
_options = SettingsMgr()
options = OptionsObject(_options)
def option(name, default, helpstr):
return options.setdefault(name, default, helpstr)
theme = option
def replayableOption(optname, default, helpstr):
option(optname, default, helpstr).replayable = True
replayableOption('encoding', 'utf-8', 'encoding passed to codecs.open')
replayableOption('encoding_errors', 'surrogateescape', 'encoding_errors passed to codecs.open')
replayableOption('regex_flags', 'I', 'flags to pass to re.compile() [AILMSUX]')
replayableOption('default_width', 20, 'default column width')
option('wrap', False, 'wrap text to fit window width on TextSheet')
replayableOption('bulk_select_clear', False, 'clear selected rows before new bulk selections')
option('cmd_after_edit', 'go-down', 'command longname to execute after successful edit')
option('col_cache_size', 0, 'max number of cache entries in each cached column')
option('quitguard', False, 'confirm before quitting last sheet')
replayableOption('null_value', None, 'a value to be counted as null')
replayableOption('force_valid_colnames', False, 'clean column names to be valid Python identifiers')
option('debug', False, 'exit on error and display stacktrace')
option('curses_timeout', 100, 'curses timeout in ms')
theme('force_256_colors', False, 'use 256 colors even if curses reports fewer')
theme('use_default_colors', False, 'curses use default terminal colors')
disp_column_fill = ' ' # pad chars after column value
theme('disp_note_none', '⌀', 'visible contents of a cell whose value is None')
theme('disp_truncator', '…', 'indicator that the contents are only partially visible')
theme('disp_oddspace', '\u00b7', 'displayable character for odd whitespace')
theme('disp_unprintable', '.', 'substitute character for unprintables')
theme('disp_column_sep', '|', 'separator between columns')
theme('disp_keycol_sep', '\u2016', 'separator between key columns and rest of columns')
theme('disp_status_fmt', '{sheet.name}| ', 'status line prefix')
theme('disp_lstatus_max', 0, 'maximum length of left status line')
theme('disp_status_sep', ' | ', 'separator between statuses')
theme('disp_edit_fill', '_', 'edit field fill character')
theme('disp_more_left', '<', 'header note indicating more columns to the left')
theme('disp_more_right', '>', 'header note indicating more columns to the right')
theme('disp_error_val', '', 'displayed contents for computation exception')
theme('disp_ambig_width', 1, 'width to use for unicode chars marked ambiguous')
theme('color_default', 'normal', 'the default color')
theme('color_default_hdr', 'bold underline', 'color of the column headers')
theme('color_current_row', 'reverse', 'color of the cursor row')
theme('color_current_col', 'bold', 'color of the cursor column')
theme('color_current_hdr', 'bold reverse underline', 'color of the header for the cursor column')
theme('color_column_sep', '246 blue', 'color of column separators')
theme('color_key_col', '81 cyan', 'color of key columns')
theme('color_hidden_col', '8', 'color of hidden columns on metasheets')
theme('color_selected_row', '215 yellow', 'color of selected rows')
theme('color_keystrokes', 'white', 'color of input keystrokes on status line')
theme('color_status', 'bold', 'status line color')
theme('color_error', 'red', 'error message color')
theme('color_warning', 'yellow', 'warning message color')
theme('color_edit_cell', 'normal', 'cell color to use when editing cell')
theme('disp_pending', '', 'string to display in pending cells')
theme('note_pending', '⌛', 'note to display for pending cells')
theme('note_format_exc', '?', 'cell note for an exception during formatting')
theme('note_getter_exc', '!', 'cell note for an exception during computation')
theme('note_type_exc', '!', 'cell note for an exception during type conversion')
theme('note_unknown_type', '', 'cell note for unknown types in anytype column')
theme('color_note_pending', 'bold magenta', 'color of note in pending cells')
theme('color_note_type', '226 yellow', 'cell note for numeric types in anytype columns')
theme('scroll_incr', 3, 'amount to scroll with scrollwheel')
ENTER='^J'
ESC='^['
globalCommand('KEY_RESIZE', 'no-op', '')
globalCommand('q', 'quit-sheet', 'vd.sheets[1:] or options.quitguard and confirm("quit last sheet? "); vd.sheets.pop(0)')
globalCommand('gq', 'quit-all', 'vd.sheets.clear()')
globalCommand('^L', 'redraw', 'vd.scr.clear()')
globalCommand('^^', 'prev-sheet', 'vd.sheets[1:] or fail("no previous sheet"); vd.sheets[0], vd.sheets[1] = vd.sheets[1], vd.sheets[0]')
globalCommand('^Z', 'suspend', 'suspend()')
globalCommand(' ', 'exec-longname', 'exec_keystrokes(input_longname(sheet))')
bindkey('KEY_RESIZE', 'redraw')
def input_longname(sheet):
longnames = set(k for (k, obj), v in commands.iter(sheet))
return input("command name: ", completer=CompleteKey(sorted(longnames)))
# _vdtype .typetype are e.g. int, float, str, and used internally in these ways:
#
# o = typetype(val) # for interpreting raw value
# o = typetype(str) # for conversion from string (when setting)
# o = typetype() # for default value to be used when conversion fails
#
# The resulting object o must be orderable and convertible to a string for display and certain outputs (like csv).
#
# .icon is a single character that appears in the notes field of cells and column headers.
# .formatter(fmtstr, typedvalue) returns a string of the formatted typedvalue according to fmtstr.
# .fmtstr is the default fmtstr passed to .formatter.
_vdtype = collections.namedtuple('type', 'typetype icon fmtstr formatter')
def anytype(r=None):
'minimalist "any" passthrough type'
return r
anytype.__name__ = ''
def _defaultFormatter(fmtstr, typedval):
if fmtstr:
return fmtstr.format(typedval)
return str(typedval)
def vdtype(typetype, icon='', fmtstr='', formatter=_defaultFormatter):
t = _vdtype(typetype, icon, fmtstr, formatter)
typemap[typetype] = t
return t
# typemap [typetype] -> _vdtype
typemap = {}
def getType(typetype):
return typemap.get(typetype) or _vdtype(anytype, '', '', _defaultFormatter)
def typeIcon(typetype):
t = typemap.get(typetype, None)
if t:
return t.icon
vdtype(None, '∅')
vdtype(anytype, '', formatter=lambda _,v: str(v))
vdtype(str, '~', formatter=lambda _,v: v)
vdtype(int, '#', '{:.0f}')
vdtype(float, '%', '{:.02f}')
vdtype(len, '♯', '{:.0f}')
vdtype(dict, '')
vdtype(list, '')
def isNumeric(col):
return col.type in (int,len,float,currency,date)
###
def catchapply(func, *args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
exceptionCaught(e)
def error(s):
'Log an error and raise an exception.'
status(s, priority=3)
raise ExpectedException(s)
def fail(s):
status(s, priority=2)
raise ExpectedException(s)
def warning(s):
status(s, priority=1)
def status(*args, **kwargs):
return vd().status(*args, **kwargs)
def debug(*args, **kwargs):
if options.debug:
return vd().status(*args, **kwargs)
def input(*args, **kwargs):
return vd().input(*args, **kwargs)
def rotate_range(n, idx, reverse=False):
if reverse:
rng = range(idx-1, -1, -1)
rng2 = range(n-1, idx-1, -1)
else:
rng = range(idx+1, n)
rng2 = range(0, idx+1)
wrapped = False
with Progress(total=n) as prog:
for r in itertools.chain(rng, rng2):
prog.addProgress(1)
if not wrapped and r in rng2:
status('search wrapped')
wrapped = True
yield r
def clean_to_id(s): # [Nas Banov] https://stackoverflow.com/a/3305731
return re.sub(r'\W|^(?=\d)', '_', str(s))
def middleTruncate(s, w):
if len(s) <= w:
return s
return s[:w] + options.disp_truncator + s[-w:]
def composeStatus(msgparts, n):
msg = '; '.join(wrmap(str, msgparts))
if n > 1:
msg = '[%sx] %s' % (n, msg)
return msg
def exceptionCaught(e, **kwargs):
return vd().exceptionCaught(e, **kwargs)
def stacktrace(e=None):
if not e:
return traceback.format_exc().strip().splitlines()
return traceback.format_exception_only(type(e), e)
def chooseOne(choices):
'Return one of `choices` elements (if list) or values (if dict).'
ret = chooseMany(choices)
assert len(ret) == 1, 'need one choice only'
return ret[0]
def chooseMany(choices):
'Return list of `choices` elements (if list) or values (if dict).'
if isinstance(choices, dict):
choosed = input('/'.join(choices.keys()) + ': ', completer=CompleteKey(choices)).split()
return [choices[c] for c in choosed]
else:
return input('/'.join(str(x) for x in choices) + ': ', completer=CompleteKey(choices)).split()
def regex_flags():
'Return flags to pass to regex functions from options'
return sum(getattr(re, f.upper()) for f in options.regex_flags)
def sync(expectedThreads=0):
vd().sync(expectedThreads)
# define @asyncthread for potentially long-running functions
# when function is called, instead launches a thread
def asyncthread(func):
'Function decorator, to make calls to `func()` spawn a separate thread if available.'
@functools.wraps(func)
def _execAsync(*args, **kwargs):
return vd().execAsync(func, *args, **kwargs)
return _execAsync
def asynccache(key=lambda *args, **kwargs: str(args)+str(kwargs)):
def _decorator(func):
'Function decorator, so first call to `func()` spawns a separate thread. Calls return the Thread until the wrapped function returns; subsequent calls return the cached return value.'
d = {} # per decoration cache
def _func(k, *args, **kwargs):
d[k] = func(*args, **kwargs)
@functools.wraps(func)
def _execAsync(*args, **kwargs):
k = key(*args, **kwargs)
if k not in d:
d[k] = vd().execAsync(_func, k, *args, **kwargs)
return d.get(k)
return _execAsync
return _decorator
class Progress:
def __init__(self, iterable=None, gerund="", total=None, sheet=None):
self.iterable = iterable
self.total = total if total is not None else len(iterable)
self.sheet = sheet if sheet else getattr(threading.current_thread(), 'sheet', None)
self.gerund = gerund
self.made = 0
def __enter__(self):
if self.sheet:
self.sheet.progresses.append(self)
return self
def addProgress(self, n):
self.made += n
return True
def __exit__(self, exc_type, exc_val, tb):
if self.sheet:
self.sheet.progresses.remove(self)
def __iter__(self):
with self as prog:
for item in self.iterable:
yield item
self.made += 1
@asyncthread
def _async_deepcopy(vs, newlist, oldlist):
for r in Progress(oldlist, 'copying'):
newlist.append(deepcopy(r))
def async_deepcopy(vs, rowlist):
ret = []
_async_deepcopy(vs, ret, rowlist)
return ret
class VisiData:
allPrefixes = 'gz' # embig'g'en, 'z'mallify
def __call__(self):
return self
def __init__(self):
self.sheets = [] # list of BaseSheet; all sheets on the sheet stack
self.allSheets = weakref.WeakKeyDictionary() # [BaseSheet] -> sheetname (all non-precious sheets ever pushed)
self.statuses = collections.OrderedDict() # (priority, statusmsg) -> num_repeats; shown until next action
self.statusHistory = [] # list of [priority, statusmsg, repeats] for all status messages ever
self.lastErrors = []
self.lastInputs = collections.defaultdict(collections.OrderedDict) # [input_type] -> prevInputs
self.keystrokes = ''
self.inInput = False
self.prefixWaiting = False
self.scr = None # curses scr
self.hooks = collections.defaultdict(list) # [hookname] -> list(hooks)
self.mousereg = []
self.threads = [] # all long-running threads, including main and finished
self.addThread(threading.current_thread(), endTime=0)
self.addHook('rstatus', lambda sheet,self=self: (self.keystrokes, 'color_keystrokes'))
self.addHook('rstatus', self.rightStatus)
@property
def sheet(self):
return self.sheets[0] if self.sheets else None
def getSheet(self, sheetname):
matchingSheets = [x for x in vd().sheets if x.name == sheetname]
if matchingSheets:
if len(matchingSheets) > 1:
status('more than one sheet named "%s"' % sheetname)
return matchingSheets[0]
if sheetname == 'options':
vs = self.optionsSheet
vs.reload()
vs.vd = vd()
return vs
def status(self, *args, priority=0):
'Add status message to be shown until next action.'
k = (priority, args)
self.statuses[k] = self.statuses.get(k, 0) + 1
if self.statusHistory:
prevpri, prevargs, prevn = self.statusHistory[-1]
if prevpri == priority and prevargs == args:
self.statusHistory[-1][2] += 1
return True
self.statusHistory.append([priority, args, 1])
return True
def addHook(self, hookname, hookfunc):
'Add hookfunc by hookname, to be called by corresponding `callHook`.'
self.hooks[hookname].insert(0, hookfunc)
def callHook(self, hookname, *args, **kwargs):
'Call all functions registered with `addHook` for the given hookname.'
r = []
for f in self.hooks[hookname]:
try:
r.append(f(*args, **kwargs))
except Exception as e:
exceptionCaught(e)
return r
def addThread(self, t, endTime=None):
t.startTime = time.process_time()
t.endTime = endTime
t.status = ''
t.profile = None
t.exception = None
self.threads.append(t)
def execAsync(self, func, *args, **kwargs):
'Execute `func(*args, **kwargs)` in a separate thread.'
thread = threading.Thread(target=self.toplevelTryFunc, daemon=True, args=(func,)+args, kwargs=kwargs)
self.addThread(thread)
if self.sheets:
currentSheet = self.sheets[0]
currentSheet.currentThreads.append(thread)
else:
currentSheet = None
thread.sheet = currentSheet
thread.start()
return thread
@staticmethod
def toplevelTryFunc(func, *args, **kwargs):
'Thread entry-point for `func(*args, **kwargs)` with try/except wrapper'
t = threading.current_thread()
t.name = func.__name__
ret = None
try:
ret = func(*args, **kwargs)
except EscapeException as e: # user aborted
t.status += 'aborted by user'
status('%s aborted' % t.name, priority=2)
except Exception as e:
t.exception = e
exceptionCaught(e)
if t.sheet:
t.sheet.currentThreads.remove(t)
return ret
@property
def unfinishedThreads(self):
'A list of unfinished threads (those without a recorded `endTime`).'
return [t for t in self.threads if getattr(t, 'endTime', None) is None]
def checkForFinishedThreads(self):
'Mark terminated threads with endTime.'
for t in self.unfinishedThreads:
if not t.is_alive():
t.endTime = time.process_time()
if getattr(t, 'status', None) is None:
t.status = 'ended'
def sync(self, expectedThreads=0):
'Wait for all but expectedThreads async threads to finish.'
while len(self.unfinishedThreads) > expectedThreads:
time.sleep(.3)
self.checkForFinishedThreads()
def refresh(self):
Sheet.visibleCols.fget.cache_clear()
Sheet.keyCols.fget.cache_clear()
colors.colorcache.clear()
self.mousereg.clear()
def editText(self, y, x, w, record=True, **kwargs):
'Wrap global editText with `preedit` and `postedit` hooks.'
v = self.callHook('preedit') if record else None
if not v or v[0] is None:
with EnableCursor():
v = editText(self.scr, y, x, w, **kwargs)
else:
v = v[0]
if kwargs.get('display', True):
status('"%s"' % v)
self.callHook('postedit', v) if record else None
return v
def input(self, prompt, type='', defaultLast=False, **kwargs):
'Get user input, with history of `type`, defaulting to last history item if no input and defaultLast is True.'
if type:
histlist = list(self.lastInputs[type].keys())
ret = self._inputLine(prompt, history=histlist, **kwargs)
if ret:
self.lastInputs[type][ret] = ret
elif defaultLast:
histlist or fail("no previous input")
ret = histlist[-1]
else:
ret = self._inputLine(prompt, **kwargs)
return ret
def _inputLine(self, prompt, **kwargs):
'Add prompt to bottom of screen and get line of input from user.'
self.inInput = True
rstatuslen = self.drawRightStatus(self.scr, self.sheets[0])
attr = 0
promptlen = clipdraw(self.scr, self.windowHeight-1, 0, prompt, attr, w=self.windowWidth-rstatuslen-1)
ret = self.editText(self.windowHeight-1, promptlen, self.windowWidth-promptlen-rstatuslen-2,
attr=colors.color_edit_cell,
unprintablechar=options.disp_unprintable,
truncchar=options.disp_truncator,
**kwargs)
self.inInput = False
return ret
def getkeystroke(self, scr, vs=None):
'Get keystroke and display it on status bar.'
k = None
try:
k = scr.get_wch()
self.drawRightStatus(scr, vs or self.sheets[0]) # continue to display progress %
except curses.error:
return '' # curses timeout
if isinstance(k, str):
if ord(k) >= 32 and ord(k) != 127: # 127 == DEL or ^?
return k
k = ord(k)
return curses.keyname(k).decode('utf-8')
def exceptionCaught(self, exc=None, **kwargs):
'Maintain list of most recent errors and return most recent one.'
if isinstance(exc, ExpectedException): # already reported, don't log
return
self.lastErrors.append(stacktrace())
if kwargs.get('status', True):
status(self.lastErrors[-1][-1], priority=2) # last line of latest error
if options.debug:
raise
def onMouse(self, scr, y, x, h, w, **kwargs):
self.mousereg.append((scr, y, x, h, w, kwargs))
def getMouse(self, _scr, _x, _y, button):
for scr, y, x, h, w, kwargs in self.mousereg[::-1]:
if scr == _scr and x <= _x < x+w and y <= _y < y+h and button in kwargs:
return kwargs[button]
def drawLeftStatus(self, scr, vs):
'Draw left side of status bar.'
cattr = CursesAttr(colors.color_status)
attr = cattr.attr
error_attr = cattr.update_attr(colors.color_error, 1).attr
warn_attr = cattr.update_attr(colors.color_warning, 2).attr
sep = options.disp_status_sep
try:
lstatus = vs.leftStatus()
maxwidth = options.disp_lstatus_max
if maxwidth > 0:
lstatus = middleTruncate(lstatus, maxwidth//2)
y = self.windowHeight-1
x = clipdraw(scr, y, 0, lstatus, attr)
self.onMouse(scr, y, 0, 1, x,
BUTTON1_PRESSED='sheets',
BUTTON3_PRESSED='rename-sheet',
BUTTON3_CLICKED='rename-sheet')
one = False
for (pri, msgparts), n in sorted(self.statuses.items(), key=lambda k: -k[0][0]):
if x > self.windowWidth:
break
if one: # any messages already:
x += clipdraw(scr, y, x, sep, attr, self.windowWidth)
one = True
msg = composeStatus(msgparts, n)
if pri == 3: msgattr = error_attr
elif pri == 2: msgattr = warn_attr
elif pri == 1: msgattr = warn_attr
else: msgattr = attr
x += clipdraw(scr, y, x, msg, msgattr, self.windowWidth)
except Exception as e:
self.exceptionCaught(e)
def drawRightStatus(self, scr, vs):
'Draw right side of status bar. Return length displayed.'
rightx = self.windowWidth-1
ret = 0
for rstatcolor in self.callHook('rstatus', vs):
if rstatcolor:
try:
rstatus, coloropt = rstatcolor
rstatus = ' '+rstatus
attr = colors.get_color(coloropt).attr
statuslen = clipdraw(scr, self.windowHeight-1, rightx, rstatus, attr, rtl=True)
rightx -= statuslen
ret += statuslen
except Exception as e:
self.exceptionCaught(e)
if scr:
curses.doupdate()
return ret
def rightStatus(self, sheet):
'Compose right side of status bar.'
if sheet.currentThreads:
gerund = (' '+sheet.progresses[0].gerund) if sheet.progresses else ''
status = '%9d %2d%%%s' % (len(sheet), sheet.progressPct, gerund)
else:
status = '%9d %s' % (len(sheet), sheet.rowtype)
return status, 'color_status'
@property
def windowHeight(self):
return self.scr.getmaxyx()[0] if self.scr else 25
@property
def windowWidth(self):
return self.scr.getmaxyx()[1] if self.scr else 80
def run(self, scr):
'Manage execution of keystrokes and subsequent redrawing of screen.'
global sheet
scr.timeout(int(options.curses_timeout))
with suppress(curses.error):
curses.curs_set(0)
self.scr = scr
numTimeouts = 0
self.keystrokes = ''
while True:
if not self.sheets:
# if no more sheets, exit
return
sheet = self.sheets[0]
threading.current_thread().sheet = sheet
try:
sheet.draw(scr)
except Exception as e:
self.exceptionCaught(e)
self.drawLeftStatus(scr, sheet)
self.drawRightStatus(scr, sheet) # visible during this getkeystroke
keystroke = self.getkeystroke(scr, sheet)
if keystroke: # wait until next keystroke to clear statuses and previous keystrokes
numTimeouts = 0
if not self.prefixWaiting:
self.keystrokes = ''
self.statuses.clear()
if keystroke == 'KEY_MOUSE':
self.keystrokes = ''
clicktype = ''
try:
devid, x, y, z, bstate = curses.getmouse()
sheet.mouseX, sheet.mouseY = x, y
if bstate & curses.BUTTON_CTRL:
clicktype += "CTRL-"
bstate &= ~curses.BUTTON_CTRL
if bstate & curses.BUTTON_ALT:
clicktype += "ALT-"
bstate &= ~curses.BUTTON_ALT
if bstate & curses.BUTTON_SHIFT:
clicktype += "SHIFT-"
bstate &= ~curses.BUTTON_SHIFT
keystroke = clicktype + curses.mouseEvents.get(bstate, str(bstate))
f = self.getMouse(scr, x, y, keystroke)
if f:
if isinstance(f, str):
for cmd in f.split():
sheet.exec_keystrokes(cmd)
else:
f(y, x, keystroke)
self.keystrokes = keystroke
keystroke = ''
except curses.error:
pass
except Exception as e:
exceptionCaught(e)
self.keystrokes += keystroke
self.drawRightStatus(scr, sheet) # visible for commands that wait for input
if not keystroke: # timeout instead of keypress
pass
elif keystroke == '^Q':
return self.lastErrors and '\n'.join(self.lastErrors[-1])
elif bindkeys._get(self.keystrokes):
sheet.exec_keystrokes(self.keystrokes)
self.prefixWaiting = False
elif keystroke in self.allPrefixes:
self.keystrokes = ''.join(sorted(set(self.keystrokes))) # prefix order/quantity does not matter
self.prefixWaiting = True
else:
status('no command for "%s"' % (self.keystrokes))
self.prefixWaiting = False
self.checkForFinishedThreads()
self.callHook('predraw')
catchapply(sheet.checkCursor)
# no idle redraw unless background threads are running
time.sleep(0) # yield to other threads which may not have started yet
if vd.unfinishedThreads:
scr.timeout(options.curses_timeout)
else:
numTimeouts += 1
if numTimeouts > 1:
scr.timeout(-1)
else:
scr.timeout(options.curses_timeout)
def replace(self, vs):
'Replace top sheet with the given sheet `vs`.'
self.sheets.pop(0)
return self.push(vs)
def remove(self, vs):
if vs in self.sheets:
self.sheets.remove(vs)
else:
fail('sheet not on stack')
def push(self, vs):
'Move given sheet `vs` to index 0 of list `sheets`.'
if vs:
vs.vd = self
if vs in self.sheets:
self.sheets.remove(vs)
self.sheets.insert(0, vs)
elif not vs.loaded:
self.sheets.insert(0, vs)
vs.reload()
vs.recalc() # set up Columns
else:
self.sheets.insert(0, vs)
if vs.precious and vs not in vs.vd.allSheets:
vs.vd.allSheets[vs] = vs.name
return vs
# end VisiData class
vd = VisiData()
class LazyMap:
'provides a lazy mapping to obj attributes. useful when some attributes are expensive properties.'
def __init__(self, obj):
self.obj = obj
def keys(self):
return dir(self.obj)
def __getitem__(self, k):
if k not in dir(self.obj):
raise KeyError(k)
return getattr(self.obj, k)
def __setitem__(self, k, v):
setattr(self.obj, k, v)
class CompleteExpr:
def __init__(self, sheet=None):
self.sheet = sheet
def __call__(self, val, state):
i = len(val)-1
while val[i:].isidentifier() and i >= 0:
i -= 1
if i < 0:
base = ''
partial = val
elif val[i] == '.': # no completion of attributes
return None
else:
base = val[:i+1]
partial = val[i+1:]
varnames = []
varnames.extend(sorted((base+col.name) for col in self.sheet.columns if col.name.startswith(partial)))
varnames.extend(sorted((base+x) for x in globals() if x.startswith(partial)))
return varnames[state%len(varnames)]
class CompleteKey:
def __init__(self, items):
self.items = items
def __call__(self, val, state):
opts = [x for x in self.items if x.startswith(val)]
return opts[state%len(opts)]
# higher precedence color overrides lower; all non-color attributes combine
# coloropt is the color option name (like 'color_error')
# func(sheet,col,row,value) should return a true value if coloropt should be applied
# if coloropt is None, func() should return a coloropt (or None) instead
RowColorizer = collections.namedtuple('RowColorizer', 'precedence coloropt func')
CellColorizer = collections.namedtuple('CellColorizer', 'precedence coloropt func')
ColumnColorizer = collections.namedtuple('ColumnColorizer', 'precedence coloropt func')
class BaseSheet:
_rowtype = object # callable (no parms) that returns new empty item
_coltype = None # callable (no parms) that returns new settable view into that item
rowtype = 'objects' # one word, plural, describing the items
precious = True # False for a few discardable metasheets
def __init__(self, name, **kwargs):
self.name = name
self.vd = vd()
# for progress bar
self.progresses = [] # list of Progress objects
# track all async threads from sheet
self.currentThreads = []
self.__dict__.update(kwargs)
@classmethod
def addCommand(cls, keystrokes, longname, execstr, helpstr=''):
commands.set(longname, Command(longname, execstr), cls)
if keystrokes:
bindkeys.set(keystrokes, longname, cls)
@classmethod
def bindkey(cls, keystrokes, longname):
oldlongname = bindkeys._get(keystrokes, cls)
if oldlongname:
warning('%s was already bound to %s' % (keystrokes, oldlongname))
bindkeys.set(keystrokes, longname, cls)
def getCommand(self, keystrokes_or_longname):
longname = bindkeys._get(keystrokes_or_longname)
try:
if longname:
return commands._get(longname) or fail('no command "%s"' % longname)
else:
return commands._get(keystrokes_or_longname) or fail('no binding for %s' % keystrokes_or_longname)
except Exception:
return None
def __bool__(self):
'an instantiated Sheet always tests true'
return True
def __len__(self):
return 0
@property
def loaded(self):
return False
def leftStatus(self):
'Compose left side of status bar for this sheet (overridable).'
return options.disp_status_fmt.format(sheet=self)
def exec_keystrokes(self, keystrokes, vdglobals=None):
return self.exec_command(self.getCommand(keystrokes), vdglobals, keystrokes=keystrokes)
def exec_command(self, cmd, args='', vdglobals=None, keystrokes=None):
"Execute `cmd` tuple with `vdglobals` as globals and this sheet's attributes as locals. Returns True if user cancelled."
global sheet
sheet = vd.sheets[0]
if not cmd:
debug('no command "%s"' % keystrokes)
return True
if isinstance(cmd, CommandLog):
cmd.replay()
return False
escaped = False
err = ''
if vdglobals is None:
vdglobals = getGlobals()
if not self.vd:
self.vd = vd()
self.sheet = self
try:
self.vd.callHook('preexec', self, cmd, '', keystrokes)
exec(cmd.execstr, vdglobals, LazyMap(self))
except EscapeException as e: # user aborted
status('aborted')
escaped = True
except Exception as e:
debug(cmd.execstr)
err = self.vd.exceptionCaught(e)
escaped = True
try:
self.vd.callHook('postexec', self.vd.sheets[0] if self.vd.sheets else None, escaped, err)
except Exception:
self.vd.exceptionCaught(e)
catchapply(self.checkCursor)
self.vd.refresh()
return escaped
@property
def name(self):
return self._name or ''
@name.setter
def name(self, name):
'Set name without spaces.'
self._name = name.strip().replace(' ', '_')
@property
def progressMade(self):
return sum(prog.made for prog in self.progresses)
@property
def progressTotal(self):
return sum(prog.total for prog in self.progresses)
@property
def progressPct(self):
'Percent complete as indicated by async actions.'
if self.progressTotal != 0:
return int(self.progressMade*100/self.progressTotal)
return 0
def recalc(self):
'Clear any calculated value caches.'
pass
def draw(self, scr):
error('no draw')
def reload(self):
error('no reload')
def checkCursor(self):
pass
def newRow(self):
return type(self)._rowtype()
BaseSheet.addCommand('^R', 'reload-sheet', 'reload(); status("reloaded")')
class CursesAttr:
def __init__(self, attr=0, precedence=-1):
self.attributes = attr & ~curses.A_COLOR
self.color = attr & curses.A_COLOR
self.precedence = precedence
def __str__(self):
a = self.attr
ret = set()
for k in dir(curses):
v = getattr(curses, k)
if v == 0:
pass
elif k.startswith('A_'):
if self.attributes & v == v:
ret.add(k)
elif k.startswith('COLOR_'):
if self.color == v:
ret.add(k)
return (' '.join(k for k in ret) or str(self.color)) + ' %0X' % self.attr
@property
def attr(self):
'the composed curses attr'
return self.color | self.attributes
def update_attr(self, newattr, newprec=None):
if isinstance(newattr, int):
newattr = CursesAttr(newattr)
ret = copy(self)
if newprec is None:
newprec = newattr.precedence
ret.attributes |= newattr.attributes
if not ret.color or newprec > ret.precedence:
if newattr.color:
ret.color = newattr.color
ret.precedence = newprec
return ret
class Sheet(BaseSheet):
'Base class for all tabular sheets.'
_rowtype = lambda: defaultdict(lambda: None)
rowtype = 'rows'
columns = [] # list of Column
colorizers = [ # list of Colorizer
CellColorizer(2, 'color_default_hdr', lambda s,c,r,v: r is None),
ColumnColorizer(2, 'color_current_col', lambda s,c,r,v: c is s.cursorCol),
ColumnColorizer(1, 'color_key_col', lambda s,c,r,v: c in s.keyCols),
CellColorizer(0, 'color_default', lambda s,c,r,v: True),
# RowColorizer(-1, 'color_column_sep', lambda s,c,r,v: c is None),
RowColorizer(2, 'color_selected_row', lambda s,c,r,v: s.isSelected(r)),
RowColorizer(1, 'color_error', lambda s,c,r,v: isinstance(r, (Exception, TypedExceptionWrapper))),
]
nKeys = 0 # columns[:nKeys] are key columns
def __init__(self, name, **kwargs):
super().__init__(name, **kwargs)
self.rows = tuple() # list of opaque row objects (tuple until first reload)
self.cursorRowIndex = 0 # absolute index of cursor into self.rows
self.cursorVisibleColIndex = 0 # index of cursor into self.visibleCols
self.topRowIndex = 0 # cursorRowIndex of topmost row
self.leftVisibleColIndex = 0 # cursorVisibleColIndex of leftmost column
self.rightVisibleColIndex = 0
# as computed during draw()
self.rowLayout = {} # [rowidx] -> y
self.visibleColLayout = {} # [vcolidx] -> (x, w)
# list of all columns in display order
self.columns = kwargs.get('columns') or [copy(c) for c in self.columns] or [Column('')]
self.recalc() # set .sheet on columns and start caches
self.setKeys(self.columns[:self.nKeys]) # initial list of key columns
self._selectedRows = {} # id(row) -> row
self.__dict__.update(kwargs) # also done earlier in BaseSheet.__init__
def __len__(self):
return self.nRows
@property
def loaded(self):
if self.rows == tuple():
self.rows = list()
return False
return True
def addColorizer(self, c):
self.colorizers.append(c)
@functools.lru_cache()
def getColorizers(self):
_colorizers = set()
def allParents(cls):
yield from cls.__bases__
for b in cls.__bases__:
yield from allParents(b)
for b in [self] + list(allParents(self.__class__)):
for c in getattr(b, 'colorizers', []):
_colorizers.add(c)
return sorted(_colorizers, key=lambda x: x.precedence, reverse=True)
def colorize(self, col, row, value=None):
'Returns curses attribute for the given col/row/value'
# colorstack = tuple(c.coloropt for c in self.getColorizers() if wrapply(c.func, self, col, row, value))
colorstack = []
for colorizer in self.getColorizers():
try:
r = colorizer.func(self, col, row, value)
if r:
colorstack.append(colorizer.coloropt if colorizer.coloropt else r)
except Exception as e:
exceptionCaught(e)
return colors.resolve_colors(tuple(colorstack))
def addRow(self, row, index=None):
if index is None:
self.rows.append(row)
else:
self.rows.insert(index, row)
return row
def column(self, colregex):
'Return first column whose Column.name matches colregex.'
for c in self.columns:
if re.search(colregex, c.name, regex_flags()):
return c
def recalc(self):
'Clear caches and set col.sheet to this sheet for all columns.'
for c in self.columns:
c.recalc(self)
def reload(self):
'Loads rows and/or columns. Override in subclass.'
self.rows = []
for r in self.iterload():
self.addRow(r)
def iterload(self):
'Override this generator for loading, if columns can be predefined.'
for row in []:
yield row
def __copy__(self):
'copy sheet design (no rows). deepcopy columns so their attributes (width, type, name) may be adjusted independently.'
cls = self.__class__
ret = cls.__new__(cls)
ret.__dict__.update(self.__dict__)
ret.rows = [] # a fresh list without incurring any overhead
ret.columns = [copy(c) for c in self.keyCols]
ret.setKeys(ret.columns)
ret.columns.extend(copy(c) for c in self.columns if c not in self.keyCols)
ret.recalc() # set .sheet on columns
ret._selectedRows = {}
ret.topRowIndex = ret.cursorRowIndex = 0
ret.progresses = []
ret.currentThreads = []
ret.precious = True # copies can be precious even if originals aren't
return ret
def __deepcopy__(self, memo):
'same as __copy__'
ret = self.__copy__()
memo[id(self)] = ret
return ret
def deleteBy(self, func):
'Delete rows for which func(row) is true. Returns number of deleted rows.'
oldrows = copy(self.rows)
oldidx = self.cursorRowIndex
ndeleted = 0
row = None # row to re-place cursor after
while oldidx < len(oldrows):
if not func(oldrows[oldidx]):
row = self.rows[oldidx]
break
oldidx += 1
self.rows.clear()
for r in Progress(oldrows, 'deleting'):
if not func(r):
self.rows.append(r)
if r is row:
self.cursorRowIndex = len(self.rows)-1
else:
ndeleted += 1
status('deleted %s %s' % (ndeleted, self.rowtype))
return ndeleted
@asyncthread
def deleteSelected(self):
'Delete all selected rows.'
ndeleted = self.deleteBy(self.isSelected)
nselected = len(self._selectedRows)
self._selectedRows.clear()
if ndeleted != nselected:
error('expected %s' % nselected)
@asyncthread
def delete(self, rows):
rowdict = {id(r): r for r in rows}
ndeleted = self.deleteBy(lambda r,rowdict=rowdict: id(r) in rowdict)
nrows = len(rows)
if ndeleted != nrows:
error('expected %s' % nrows)
def __repr__(self):
return self.name
def evalexpr(self, expr, row=None):
return eval(expr, getGlobals(), LazyMapRow(self, row) if row is not None else None)
def inputExpr(self, prompt, *args, **kwargs):
return input(prompt, "expr", *args, completer=CompleteExpr(self), **kwargs)
@property
def nVisibleRows(self):
'Number of visible rows at the current window height.'
return self.vd.windowHeight-2
@property
def cursorCol(self):
'Current Column object.'
return self.visibleCols[self.cursorVisibleColIndex]
@property
def cursorRow(self):
'The row object at the row cursor.'
return self.rows[self.cursorRowIndex]
@property
def visibleRows(self): # onscreen rows
'List of rows onscreen. '
return self.rows[self.topRowIndex:self.topRowIndex+self.nVisibleRows]
@property
@functools.lru_cache() # cache for perf reasons on wide sheets. cleared in .refresh()
def visibleCols(self): # non-hidden cols
'List of `Column` which are not hidden.'
return self.keyCols + [c for c in self.columns if not c.hidden and not c.keycol]
def visibleColAtX(self, x):
for vcolidx, (colx, w) in self.visibleColLayout.items():
if colx <= x <= colx+w:
return vcolidx
error('no visible column at x=%d' % x)
@property
@functools.lru_cache() # cache for perf reasons on wide sheets. cleared in .refresh()
def keyCols(self):
'Cached list of visible key columns (Columns with .key=True)'
return [c for c in self.columns if c.keycol and not c.hidden]
@property
def cursorColIndex(self):
'Index of current column into `columns`. Linear search; prefer `cursorCol` or `cursorVisibleColIndex`.'
return self.columns.index(self.cursorCol)
@property
def nonKeyVisibleCols(self):
'All columns which are not keysList of unhidden non-key columns.'
return [c for c in self.columns if not c.hidden and c not in self.keyCols]
@property
def keyColNames(self):
'String of key column names, for SheetsSheet convenience.'
return ' '.join(c.name for c in self.keyCols)
@property
def cursorCell(self):
'Displayed value (DisplayWrapper) at current row and column.'
return self.cursorCol.getCell(self.cursorRow)
@property
def cursorDisplay(self):
'Displayed value (DisplayWrapper.display) at current row and column.'
return self.cursorCol.getDisplayValue(self.cursorRow)
@property
def cursorTypedValue(self):
'Typed value at current row and column.'
return self.cursorCol.getTypedValue(self.cursorRow)
@property
def cursorValue(self):
'Raw value at current row and column.'
return self.cursorCol.getValue(self.cursorRow)
@property
def statusLine(self):
'String of row and column stats.'
rowinfo = 'row %d/%d (%d selected)' % (self.cursorRowIndex, self.nRows, len(self._selectedRows))
colinfo = 'col %d/%d (%d visible)' % (self.cursorColIndex, self.nCols, len(self.visibleCols))
return '%s %s' % (rowinfo, colinfo)
@property
def nRows(self):
'Number of rows on this sheet.'
return len(self.rows)
@property
def nCols(self):
'Number of columns on this sheet.'
return len(self.columns)
@property
def nVisibleCols(self):
'Number of visible columns on this sheet.'
return len(self.visibleCols)
## selection code
def isSelected(self, row):
'True if given row is selected. O(log n).'
return id(row) in self._selectedRows
@asyncthread
def toggle(self, rows):
'Toggle selection of given `rows`.'
for r in Progress(rows, 'toggling', total=len(self.rows)):
if not self.unselectRow(r):
self.selectRow(r)
def selectRow(self, row):
'Select given row. O(log n)'
self._selectedRows[id(row)] = row
def unselectRow(self, row):
'Unselect given row, return True if selected; else return False. O(log n)'
if id(row) in self._selectedRows:
del self._selectedRows[id(row)]
return True
else:
return False
@asyncthread
def select(self, rows, status=True, progress=True):
"Bulk select given rows. Don't show progress if progress=False; don't show status if status=False."
before = len(self._selectedRows)
if options.bulk_select_clear:
self._selectedRows.clear()
for r in (Progress(rows, 'selecting') if progress else rows):
self.selectRow(r)
if status:
if options.bulk_select_clear:
msg = 'selected %s %s%s' % (len(self._selectedRows), self.rowtype, ' instead' if before > 0 else '')
else:
msg = 'selected %s%s %s' % (len(self._selectedRows)-before, ' more' if before > 0 else '', self.rowtype)
vd.status(msg)
@asyncthread
def unselect(self, rows, status=True, progress=True):
"Unselect given rows. Don't show progress if progress=False; don't show status if status=False."
before = len(self._selectedRows)
for r in (Progress(rows, 'unselecting') if progress else rows):
self.unselectRow(r)
if status:
vd().status('unselected %s/%s %s' % (before-len(self._selectedRows), before, self.rowtype))
def selectByIdx(self, rowIdxs):
'Select given row indexes, without progress bar.'
self.select((self.rows[i] for i in rowIdxs), progress=False)
def unselectByIdx(self, rowIdxs):
'Unselect given row indexes, without progress bar.'
self.unselect((self.rows[i] for i in rowIdxs), progress=False)
def gatherBy(self, func):
'Generate only rows for which the given func returns True.'
for i in rotate_range(len(self.rows), self.cursorRowIndex):
try:
r = self.rows[i]
if func(r):
yield r
except Exception:
pass
@asyncthread
def orderBy(self, *cols, **kwargs):
try:
with Progress(self.rows, 'sorting') as prog:
# must not reassign self.rows: use .sort() instead of sorted()
self.rows.sort(key=lambda r,cols=cols,prog=prog: prog.addProgress(1) and tuple(c.getTypedValueNoExceptions(r) for c in cols), **kwargs)
except TypeError as e:
status('sort incomplete due to TypeError; change column type')
exceptionCaught(e, status=False)
@property
def selectedRows(self):
'List of selected rows in sheet order. [O(nRows*log(nSelected))]'
if len(self._selectedRows) <= 1:
return list(self._selectedRows.values())
return [r for r in self.rows if id(r) in self._selectedRows]
## end selection code
def cursorDown(self, n=1):
'Move cursor down `n` rows (or up if `n` is negative).'
self.cursorRowIndex += n
def cursorRight(self, n=1):
'Move cursor right `n` visible columns (or left if `n` is negative).'
self.cursorVisibleColIndex += n
self.calcColLayout()
def pageLeft(self):
'''Redraw page one screen to the left.
Note: keep the column cursor in the same general relative position:
- if it is on the furthest right column, then it should stay on the
furthest right column if possible
- likewise on the left or in the middle
So really both the `leftIndex` and the `cursorIndex` should move in
tandem until things are correct.'''
targetIdx = self.leftVisibleColIndex # for rightmost column
firstNonKeyVisibleColIndex = self.visibleCols.index(self.nonKeyVisibleCols[0])
while self.rightVisibleColIndex != targetIdx and self.leftVisibleColIndex > firstNonKeyVisibleColIndex:
self.cursorVisibleColIndex -= 1
self.leftVisibleColIndex -= 1
self.calcColLayout() # recompute rightVisibleColIndex
# in case that rightmost column is last column, try to squeeze maximum real estate from screen
if self.rightVisibleColIndex == self.nVisibleCols-1:
# try to move further left while right column is still full width
while self.leftVisibleColIndex > 0:
rightcol = self.visibleCols[self.rightVisibleColIndex]
if rightcol.width > self.visibleColLayout[self.rightVisibleColIndex][1]:
# went too far
self.cursorVisibleColIndex += 1
self.leftVisibleColIndex += 1
break
else:
self.cursorVisibleColIndex -= 1
self.leftVisibleColIndex -= 1
self.calcColLayout() # recompute rightVisibleColIndex
def addColumn(self, col, index=None):
'Insert column at given index or after all columns.'
if col:
if index is None:
index = len(self.columns)
col.sheet = self
self.columns.insert(index, col)
return col
def setKeys(self, cols):
for col in cols:
col.keycol = True
def unsetKeys(self, cols):
for col in cols:
col.keycol = False
def toggleKeys(self, cols):
for col in cols:
col.keycol = not col.keycol
def rowkey(self, row):
'returns a tuple of the key for the given row'
return tuple(c.getTypedValueOrException(row) for c in self.keyCols)
def checkCursor(self):
'Keep cursor in bounds of data and screen.'
# keep cursor within actual available rowset
if self.nRows == 0 or self.cursorRowIndex <= 0:
self.cursorRowIndex = 0
elif self.cursorRowIndex >= self.nRows:
self.cursorRowIndex = self.nRows-1
if self.cursorVisibleColIndex <= 0:
self.cursorVisibleColIndex = 0
elif self.cursorVisibleColIndex >= self.nVisibleCols:
self.cursorVisibleColIndex = self.nVisibleCols-1
if self.topRowIndex <= 0:
self.topRowIndex = 0
elif self.topRowIndex > self.nRows-1:
self.topRowIndex = self.nRows-1
# (x,y) is relative cell within screen viewport
x = self.cursorVisibleColIndex - self.leftVisibleColIndex
y = self.cursorRowIndex - self.topRowIndex + 1 # header
# check bounds, scroll if necessary
if y < 1:
self.topRowIndex = self.cursorRowIndex
elif y > self.nVisibleRows:
self.topRowIndex = self.cursorRowIndex-self.nVisibleRows+1
if x <= 0:
self.leftVisibleColIndex = self.cursorVisibleColIndex
else:
while True:
if self.leftVisibleColIndex == self.cursorVisibleColIndex: # not much more we can do
break
self.calcColLayout()
mincolidx, maxcolidx = min(self.visibleColLayout.keys()), max(self.visibleColLayout.keys())
if self.cursorVisibleColIndex < mincolidx:
self.leftVisibleColIndex -= max((self.cursorVisibleColIndex - mincolid)//2, 1)
continue
elif self.cursorVisibleColIndex > maxcolidx:
self.leftVisibleColIndex += max((maxcolidx - self.cursorVisibleColIndex)//2, 1)
continue
cur_x, cur_w = self.visibleColLayout[self.cursorVisibleColIndex]
if cur_x+cur_w < self.vd.windowWidth: # current columns fit entirely on screen
break
self.leftVisibleColIndex += 1 # once within the bounds, walk over one column at a time
def calcColLayout(self):
'Set right-most visible column, based on calculation.'
minColWidth = len(options.disp_more_left)+len(options.disp_more_right)
sepColWidth = len(options.disp_column_sep)
winWidth = self.vd.windowWidth
self.visibleColLayout = {}
x = 0
vcolidx = 0
for vcolidx in range(0, self.nVisibleCols):
col = self.visibleCols[vcolidx]
if col.width is None and len(self.visibleRows) > 0:
# handle delayed column width-finding
col.width = col.getMaxWidth(self.visibleRows)+minColWidth
if vcolidx != self.nVisibleCols-1: # let last column fill up the max width
col.width = min(col.width, options.default_width)
width = col.width if col.width is not None else options.default_width
if col in self.keyCols:
width = max(width, 1) # keycols must all be visible
if col in self.keyCols or vcolidx >= self.leftVisibleColIndex: # visible columns
self.visibleColLayout[vcolidx] = [x, min(width, winWidth-x)]
x += width+sepColWidth
if x > winWidth-1:
break
self.rightVisibleColIndex = vcolidx
def drawColHeader(self, scr, y, vcolidx):
'Compose and draw column header for given vcolidx.'
col = self.visibleCols[vcolidx]
# hdrattr highlights whole column header
# sepattr is for header separators and indicators
sepattr = colors.color_column_sep
hdrattr = self.colorize(col, None)
if vcolidx == self.cursorVisibleColIndex:
hdrattr = hdrattr.update_attr(colors.color_current_hdr, 2)
C = options.disp_column_sep
if (self.keyCols and col is self.keyCols[-1]) or vcolidx == self.rightVisibleColIndex:
C = options.disp_keycol_sep
x, colwidth = self.visibleColLayout[vcolidx]
# ANameTC
T = getType(col.type).icon
if T is None: # still allow icon to be explicitly non-displayed ''
T = '?'
N = ' ' + col.name # save room at front for LeftMore
if len(N) > colwidth-1:
N = N[:colwidth-len(options.disp_truncator)] + options.disp_truncator
clipdraw(scr, y, x, N, hdrattr.attr, colwidth)
clipdraw(scr, y, x+colwidth-len(T), T, hdrattr.attr, len(T))
vd.onMouse(scr, y, x, 1, colwidth, BUTTON3_RELEASED='rename-col')
if vcolidx == self.leftVisibleColIndex and col not in self.keyCols and self.nonKeyVisibleCols.index(col) > 0:
A = options.disp_more_left
scr.addstr(y, x, A, sepattr)
if C and x+colwidth+len(C) < self.vd.windowWidth:
scr.addstr(y, x+colwidth, C, sepattr)
def isVisibleIdxKey(self, vcolidx):
'Return boolean: is given column index a key column?'
return self.visibleCols[vcolidx] in self.keyCols
def draw(self, scr):
'Draw entire screen onto the `scr` curses object.'
numHeaderRows = 1
scr.erase() # clear screen before every re-draw
vd().refresh()
if not self.columns:
return
color_current_row = CursesAttr(colors.color_current_row, 5)
disp_column_sep = options.disp_column_sep
rowattrs = {} # [rowidx] -> attr
colattrs = {} # [colidx] -> attr
isNull = isNullFunc()
self.rowLayout = {}
self.calcColLayout()
vcolidx = 0
rows = list(self.rows[self.topRowIndex:self.topRowIndex+self.nVisibleRows])
for vcolidx, colinfo in sorted(self.visibleColLayout.items()):
x, colwidth = colinfo
col = self.visibleCols[vcolidx]
if x < self.vd.windowWidth: # only draw inside window
headerRow = 0
self.drawColHeader(scr, headerRow, vcolidx)
y = headerRow + numHeaderRows
for rowidx in range(0, min(len(rows), self.nVisibleRows)):
dispRowIdx = self.topRowIndex + rowidx
if dispRowIdx >= self.nRows:
break
self.rowLayout[dispRowIdx] = y
row = rows[rowidx]
cellval = col.getCell(row, colwidth-1)
try:
if isNull(cellval.value):
cellval.note = options.disp_note_none
cellval.notecolor = 'color_note_type'
except TypeError:
pass
attr = self.colorize(col, row, cellval)
# sepattr is the attr between cell/columns
rowattr = rowattrs.get(rowidx)
if rowattr is None:
rowattr = rowattrs[rowidx] = self.colorize(None, row)
sepattr = rowattr
# must apply current row here, because this colorization requires cursorRowIndex
if dispRowIdx == self.cursorRowIndex:
attr = attr.update_attr(color_current_row)
sepattr = sepattr.update_attr(color_current_row)
note = getattr(cellval, 'note', None)
if note:
noteattr = attr.update_attr(colors.get_color(cellval.notecolor), 10)
clipdraw(scr, y, x+colwidth-len(note), note, noteattr.attr, len(note))
clipdraw(scr, y, x, disp_column_fill+cellval.display, attr.attr, colwidth-(1 if note else 0))
vd.onMouse(scr, y, x, 1, colwidth, BUTTON3_RELEASED='edit-cell')
sepchars = disp_column_sep
if (self.keyCols and col is self.keyCols[-1]) or vcolidx == self.rightVisibleColIndex:
sepchars = options.disp_keycol_sep
if x+colwidth+len(sepchars) <= self.vd.windowWidth:
scr.addstr(y, x+colwidth, sepchars, sepattr.attr)
y += 1
if vcolidx+1 < self.nVisibleCols:
scr.addstr(headerRow, self.vd.windowWidth-2, options.disp_more_right, colors.color_column_sep)
catchapply(self.checkCursor)
def editCell(self, vcolidx=None, rowidx=None, **kwargs):
'Call `editText` at its place on the screen. Returns the new value, properly typed'
if vcolidx is None:
vcolidx = self.cursorVisibleColIndex
x, w = self.visibleColLayout.get(vcolidx, (0, 0))
col = self.visibleCols[vcolidx]
if rowidx is None:
rowidx = self.cursorRowIndex
if rowidx < 0: # header
y = 0
currentValue = col.name
else:
y = self.rowLayout.get(rowidx, 0)
currentValue = col.getDisplayValue(self.rows[self.cursorRowIndex])
editargs = dict(value=currentValue,
fillchar=options.disp_edit_fill,
truncchar=options.disp_truncator)
editargs.update(kwargs) # update with user-specified args
r = self.vd.editText(y, x, w, **editargs)
if rowidx >= 0: # if not header
r = col.type(r) # convert input to column type, let exceptions be raised
return r
Sheet.addCommand(None, 'go-left', 'cursorRight(-1)'),
Sheet.addCommand(None, 'go-down', 'cursorDown(+1)'),
Sheet.addCommand(None, 'go-up', 'cursorDown(-1)'),
Sheet.addCommand(None, 'go-right', 'cursorRight(+1)'),
Sheet.addCommand(None, 'next-page', 'cursorDown(nVisibleRows); sheet.topRowIndex += nVisibleRows'),
Sheet.addCommand(None, 'prev-page', 'cursorDown(-nVisibleRows); sheet.topRowIndex -= nVisibleRows'),
Sheet.addCommand(None, 'go-leftmost', 'sheet.cursorVisibleColIndex = sheet.leftVisibleColIndex = 0'),
Sheet.addCommand(None, 'go-top', 'sheet.cursorRowIndex = sheet.topRowIndex = 0'),
Sheet.addCommand(None, 'go-bottom', 'sheet.cursorRowIndex = len(rows); sheet.topRowIndex = cursorRowIndex-nVisibleRows'),
Sheet.addCommand(None, 'go-rightmost', 'sheet.leftVisibleColIndex = len(visibleCols)-1; pageLeft(); sheet.cursorVisibleColIndex = len(visibleCols)-1'),
Sheet.addCommand('BUTTON1_PRESSED', 'go-mouse', 'sheet.cursorRowIndex=topRowIndex+mouseY-1; sheet.cursorVisibleColIndex=visibleColAtX(mouseX)'),
Sheet.addCommand('BUTTON1_RELEASED', 'scroll-mouse', 'sheet.topRowIndex=cursorRowIndex-mouseY+1'),
Sheet.addCommand('BUTTON4_PRESSED', 'scroll-up', 'cursorDown(options.scroll_incr); sheet.topRowIndex += options.scroll_incr'),
Sheet.addCommand('REPORT_MOUSE_POSITION', 'scroll-down', 'cursorDown(-options.scroll_incr); sheet.topRowIndex -= options.scroll_incr'),
Sheet.bindkey('BUTTON1_CLICKED', 'go-mouse')
Sheet.bindkey('BUTTON3_PRESSED', 'go-mouse')
Sheet.addCommand('^G', 'show-cursor', 'status(statusLine)'),
Sheet.addCommand('_', 'resize-col-max', 'cursorCol.toggleWidth(cursorCol.getMaxWidth(visibleRows))'),
Sheet.addCommand('z_', 'resize-col', 'cursorCol.width = int(input("set width= ", value=cursorCol.width))'),
Sheet.addCommand('-', 'hide-col', 'cursorCol.hide()'),
Sheet.addCommand('z-', 'resize-col-half', 'cursorCol.width = cursorCol.width//2'),
Sheet.addCommand('gv', 'unhide-cols', 'for c in columns: c.width = abs(c.width or 0) or c.getMaxWidth(visibleRows)'),
Sheet.addCommand('!', 'key-col', 'toggleKeys([cursorCol])'),
Sheet.addCommand('z!', 'key-col-off', 'unsetKeys([cursorCol])'),
Sheet.addCommand('g_', 'resize-cols-max', 'for c in visibleCols: c.width = c.getMaxWidth(visibleRows)'),
Sheet.addCommand('[', 'sort-asc', 'orderBy(cursorCol)'),
Sheet.addCommand(']', 'sort-desc', 'orderBy(cursorCol, reverse=True)'),
Sheet.addCommand('g[', 'sort-keys-asc', 'orderBy(*keyCols)'),
Sheet.addCommand('g]', 'sort-keys-desc', 'orderBy(*keyCols, reverse=True)'),
Sheet.addCommand('^R', 'reload-sheet', 'reload(); recalc(); status("reloaded")'),
Sheet.addCommand('e', 'edit-cell', 'cursorCol.setValues([cursorRow], editCell(cursorVisibleColIndex)); options.cmd_after_edit and sheet.exec_keystrokes(options.cmd_after_edit)'),
Sheet.addCommand('ge', 'edit-cells', 'cursorCol.setValuesTyped(selectedRows or rows, input("set selected to: ", value=cursorDisplay))'),
Sheet.addCommand('"', 'dup-selected', 'vs=copy(sheet); vs.name += "_selectedref"; vs.rows=tuple(); vs.reload=lambda vs=vs,rows=selectedRows or rows: setattr(vs, "rows", list(rows)); vd.push(vs)'),
Sheet.addCommand('g"', 'dup-rows', 'vs = copy(sheet); vs.name += "_copy"; vs.rows = list(rows); vs.select(selectedRows); vd.push(vs)'),
Sheet.addCommand('z"', 'dup-selected-deep', 'vs = deepcopy(sheet); vs.name += "_selecteddeepcopy"; vs.rows = async_deepcopy(vs, selectedRows or rows); vd.push(vs); status("pushed sheet with async deepcopy of selected rows")'),
Sheet.addCommand('gz"', 'dup-rows-deep', 'vs = deepcopy(sheet); vs.name += "_deepcopy"; vs.rows = async_deepcopy(vs, rows); vd.push(vs); status("pushed sheet with async deepcopy of all rows")'),
Sheet.addCommand('=', 'addcol-expr', 'addColumn(ColumnExpr(inputExpr("new column expr=")), index=cursorColIndex+1)'),
Sheet.addCommand('g=', 'setcol-expr', 'cursorCol.setValuesFromExpr(selectedRows or rows, inputExpr("set selected="))'),
Sheet.addCommand('V', 'view-cell', 'vd.push(TextSheet("%s[%s].%s" % (name, cursorRowIndex, cursorCol.name), cursorDisplay.splitlines()))'),
bindkey('KEY_LEFT', 'go-left')
bindkey('KEY_DOWN', 'go-down')
bindkey('KEY_UP', 'go-up')
bindkey('KEY_RIGHT', 'go-right')
bindkey('KEY_HOME', 'go-top')
bindkey('KEY_END', 'go-bottom')
bindkey('KEY_NPAGE', 'next-page')
bindkey('KEY_PPAGE', 'prev-page')
bindkey('gKEY_LEFT', 'go-leftmost'),
bindkey('gKEY_RIGHT', 'go-rightmost'),
bindkey('gKEY_UP', 'go-top'),
bindkey('gKEY_DOWN', 'go-bottom'),
def isNullFunc():
return lambda v,nulls=set([None, options.null_value]): v in nulls or isinstance(v, TypedWrapper)
@functools.total_ordering
class TypedWrapper:
def __init__(self, func, *args):
self.type = func
self.args = args
self.val = args[0] if args else None
def __bool__(self):
return False
def __str__(self):
return '%s(%s)' % (self.type.__name__, ','.join(str(x) for x in self.args))
def __lt__(self, x):
'maintain sortability; wrapped objects are always least'
return True
def __add__(self, x):
return x
def __radd__(self, x):
return x
def __hash__(self):
return hash((self.type, str(self.val)))
def __eq__(self, x):
if isinstance(x, TypedWrapper):
return self.type == x.type and self.val == x.val
class TypedExceptionWrapper(TypedWrapper):
def __init__(self, func, *args, exception=None):
TypedWrapper.__init__(self, func, *args)
self.exception = exception
self.stacktrace = stacktrace()
self.forwarded = False
def __str__(self):
return str(self.exception)
def __hash__(self):
return hash((type(self.exception), ''.join(self.stacktrace[:-1])))
def __eq__(self, x):
if isinstance(x, TypedExceptionWrapper):
return type(self.exception) is type(x.exception) and self.stacktrace[:-1] == x.stacktrace[:-1]
def forward(wr):
if isinstance(wr, TypedExceptionWrapper):
wr.forwarded = True
return wr
def wrmap(func, iterable, *args):
'Same as map(func, iterable, *args), but ignoring exceptions.'
for it in iterable:
try:
yield func(it, *args)
except Exception as e:
pass
def wrapply(func, *args, **kwargs):
'Like apply(), but which wraps Exceptions and passes through Wrappers (if first arg)'
val = args[0]
if val is None:
return TypedWrapper(func, None)
elif isinstance(val, TypedExceptionWrapper):
tew = copy(val)
tew.forwarded = True
return tew
elif isinstance(val, TypedWrapper):
return val
elif isinstance(val, Exception):
return TypedWrapper(func, *args)
try:
return func(*args, **kwargs)
except Exception as e:
e.stacktrace = stacktrace()
return TypedExceptionWrapper(func, *args, exception=e)
class Column:
def __init__(self, name='', *, type=anytype, cache=False, **kwargs):
self.sheet = None # owning Sheet, set in Sheet.addColumn
self.name = name # display visible name
self.fmtstr = '' # by default, use str()
self.type = type # anytype/str/int/float/date/func
self.getter = lambda col, row: row
self.setter = lambda col, row, value: fail(col.name+' column cannot be changed')
self.width = None # == 0 if hidden, None if auto-compute next time
self.keycol = False # is a key column
self.expr = None # Column-type-dependent parameter
self._cachedValues = collections.OrderedDict() if cache else None
for k, v in kwargs.items():
setattr(self, k, v) # instead of __dict__.update(kwargs) to invoke property.setters
def __copy__(self):
cls = self.__class__
ret = cls.__new__(cls)
ret.__dict__.update(self.__dict__)
ret.keycol = False # column copies lose their key status
if self._cachedValues is not None:
ret._cachedValues = collections.OrderedDict() # an unrelated cache for copied columns
return ret
def __deepcopy__(self, memo):
return self.__copy__() # no separate deepcopy
def recalc(self, sheet=None):
'reset column cache, attach to sheet, and reify name'
if self._cachedValues:
self._cachedValues.clear()
if sheet:
self.sheet = sheet
self.name = self._name
@property
def name(self):
return self._name or ''
@name.setter
def name(self, name):
if isinstance(name, str):
name = name.strip()
if options.force_valid_colnames:
name = clean_to_id(name)
self._name = name
@property
def fmtstr(self):
return self._fmtstr or getType(self.type).fmtstr
@fmtstr.setter
def fmtstr(self, v):
self._fmtstr = v
def format(self, typedval):
'Return displayable string of `typedval` according to `Column.fmtstr`'
if typedval is None:
return None
if isinstance(typedval, (list, tuple)):
return '[%s]' % len(typedval)
if isinstance(typedval, dict):
return '{%s}' % len(typedval)
if isinstance(typedval, bytes):
typedval = typedval.decode(options.encoding, options.encoding_errors)
return getType(self.type).formatter(self.fmtstr, typedval)
def hide(self, hide=True):
if hide:
self.width = 0
else:
self.width = abs(self.width or self.getMaxWidth(self.sheet.visibleRows))
@property
def hidden(self):
'A column is hidden if its width <= 0. (width==None means not-yet-autocomputed).'
if self.width is None:
return False
return self.width <= 0
def getValueRows(self, rows):
'Generate (val, row) for the given `rows` at this Column, excluding errors and nulls.'
f = isNullFunc()
for r in Progress(rows, 'calculating'):
try:
v = self.getTypedValue(r)
if not f(v):
yield v, r
except Exception:
pass
def getValues(self, rows):
for v, r in self.getValueRows(rows):
yield v
def calcValue(self, row):
return (self.getter)(self, row)
def getTypedValue(self, row):
'Returns the properly-typed value for the given row at this column.'
return wrapply(self.type, wrapply(self.getValue, row))
def getTypedValueOrException(self, row):
'Returns the properly-typed value for the given row at this column, or an Exception object.'
return wrapply(self.type, wrapply(self.getValue, row))
def getTypedValueNoExceptions(self, row):
'''Returns the properly-typed value for the given row at this column.
Returns the type's default value if either the getter or the type conversion fails.'''
return wrapply(self.type, wrapply(self.getValue, row))
def getValue(self, row):
'Memoize calcValue with key id(row)'
if self._cachedValues is None:
return self.calcValue(row)
k = id(row)
if k in self._cachedValues:
return self._cachedValues[k]
ret = self.calcValue(row)
self._cachedValues[k] = ret
cachesize = options.col_cache_size
if cachesize > 0 and len(self._cachedValues) > cachesize:
self._cachedValues.popitem(last=False)
return ret
def getCell(self, row, width=None):
'Return DisplayWrapper for displayable cell value.'
cellval = wrapply(self.getValue, row)
typedval = wrapply(self.type, cellval)
if isinstance(typedval, TypedWrapper):
if isinstance(cellval, TypedExceptionWrapper): # calc failed
exc = cellval.exception
if cellval.forwarded:
dispval = str(cellval) # traceback.format_exception_only(type(exc), exc)[-1].strip()
else:
dispval = options.disp_error_val
return DisplayWrapper(cellval.val, error=exc.stacktrace,
display=dispval,
note=options.note_getter_exc,
notecolor='color_error')
elif typedval.val is None: # early out for strict None
return DisplayWrapper(None, display='', # force empty display for None
note=options.disp_note_none,
notecolor='color_note_type')
elif isinstance(typedval, TypedExceptionWrapper): # calc succeeded, type failed
return DisplayWrapper(typedval.val, display=str(cellval),
error=typedval.exception.stacktrace,
note=options.note_type_exc,
notecolor='color_warning')
else:
return DisplayWrapper(typedval.val, display=str(typedval.val),
note=options.note_type_exc,
notecolor='color_warning')
elif isinstance(typedval, threading.Thread):
return DisplayWrapper(None,
display=options.disp_pending,
note=options.note_pending,
notecolor='color_note_pending')
dw = DisplayWrapper(cellval)
try:
dw.display = self.format(typedval) or ''
if width and isNumeric(self):
dw.display = dw.display.rjust(width-1)
# annotate cells with raw value type in anytype columns, except for strings
if self.type is anytype and type(cellval) is not str:
typedesc = typemap.get(type(cellval), None)
dw.note = typedesc.icon if typedesc else options.note_unknown_type
dw.notecolor = 'color_note_type'
except Exception as e: # formatting failure
e.stacktrace = stacktrace()
dw.error = e
try:
dw.display = str(cellval)
except Exception as e:
dw.display = str(e)
dw.note = options.note_format_exc
dw.notecolor = 'color_warning'
return dw
def getDisplayValue(self, row):
return self.getCell(row).display
def setValue(self, row, value):
'Set our column value on row. defaults to .setter; override in Column subclass. no type checking'
return self.setter(self, row, value)
def setValueSafe(self, row, value):
'setValue and ignore exceptions'
try:
return self.setValue(row, value)
except Exception as e:
exceptionCaught(e)
def setValues(self, rows, *values):
'Set our column value for given list of rows to `value`.'
for r, v in zip(rows, itertools.cycle(values)):
self.setValueSafe(r, v)
self.recalc()
return status('set %d cells to %d values' % (len(rows), len(values)))
def setValuesTyped(self, rows, *values):
'Set values on this column for rows, coerced to the column type. will stop on first exception in type().'
for r, v in zip(rows, itertools.cycle(self.type(val) for val in values)):
self.setValueSafe(r, v)
self.recalc()
return status('set %d cells to %d values' % (len(rows), len(values)))
@asyncthread
def setValuesFromExpr(self, rows, expr):
compiledExpr = compile(expr, '', 'eval')
for row in Progress(rows, 'setting'):
self.setValueSafe(row, self.sheet.evalexpr(compiledExpr, row))
self.recalc()
status('set %d values = %s' % (len(rows), expr))
def getMaxWidth(self, rows):
'Return the maximum length of any cell in column or its header.'
w = 0
if len(rows) > 0:
w = max(max(len(self.getDisplayValue(r)) for r in rows), len(self.name))+2
return max(w, len(self.name))
def toggleWidth(self, width):
'Change column width to either given `width` or default value.'
if self.width != width:
self.width = width
else:
self.width = int(options.default_width)
# ---- Column makers
def setitem(r, i, v): # function needed for use in lambda
r[i] = v
return True
def getattrdeep(obj, attr, *default):
'Return dotted attr (like "a.b.c") from obj, or default if any of the components are missing.'
attrs = attr.split('.')
if default:
getattr_default = lambda o,a,d=default[0]: getattr(o, a, d)
else:
getattr_default = lambda o,a: getattr(o, a)
for a in attrs[:-1]:
obj = getattr_default(obj, a)
return getattr_default(obj, attrs[-1])
def setattrdeep(obj, attr, val):
'Set dotted attr (like "a.b.c") on obj to val.'
attrs = attr.split('.')
for a in attrs[:-1]:
obj = getattr(obj, a)
setattr(obj, attrs[-1], val)
def ColumnAttr(name='', attr=None, **kwargs):
'Column using getattr/setattr of given attr.'
return Column(name,
expr=attr if attr is not None else name,
getter=lambda col,row: getattrdeep(row, col.expr),
setter=lambda col,row,val: setattrdeep(row, col.expr, val),
**kwargs)
def getitemdef(o, k, default=None):
try:
return default if o is None else o[k]
except Exception:
return default
def ColumnItem(name='', key=None, **kwargs):
'Column using getitem/setitem of given key.'
return Column(name,
expr=key if key is not None else name,
getter=lambda col,row: getitemdef(row, col.expr),
setter=lambda col,row,val: setitem(row, col.expr, val),
**kwargs)
def ArrayNamedColumns(columns):
'Return list of ColumnItems from given list of column names.'
return [ColumnItem(colname, i) for i, colname in enumerate(columns)]
def ArrayColumns(ncols):
'Return list of ColumnItems for given row length.'
return [ColumnItem('', i, width=8) for i in range(ncols)]
class SubrowColumn(Column):
def __init__(self, name, origcol, subrowidx, **kwargs):
super().__init__(name, type=origcol.type, width=origcol.width, **kwargs)
self.origcol = origcol
self.expr = subrowidx
def getValue(self, row):
subrow = row[self.expr]
if subrow is not None:
return self.origcol.getValue(subrow)
def setValue(self, row, value):
subrow = row[self.expr]
if subrow is None:
fail('no source row')
self.origcol.setValue(subrow, value)
def recalc(self, sheet=None):
Column.recalc(self, sheet)
self.origcol.recalc() # reset cache but don't change sheet
class DisplayWrapper:
def __init__(self, value, **kwargs):
self.value = value
self.__dict__.update(kwargs)
def __bool__(self):
return self.value
class ColumnEnum(Column):
'types and aggregators. row. should be kept to the values in the mapping m, and can be set by the a string key into the mapping.'
def __init__(self, name, m, default=None):
super().__init__(name)
self.mapping = m
self.default = default
def getValue(self, row):
v = getattr(row, self.name, None)
return v.__name__ if v else None
def setValue(self, row, value):
if isinstance(value, str): # first try to get the actual value from the mapping
value = self.mapping.get(value, value)
setattr(row, self.name, value or self.default)
class LazyMapRow:
'Calculate column values as needed.'
def __init__(self, sheet, row):
self.row = row
self.sheet = sheet
self._keys = [c.name for c in self.sheet.columns]
def keys(self):
return self._keys
def __getitem__(self, colid):
try:
i = self._keys.index(colid)
return self.sheet.columns[i].getTypedValue(self.row)
except ValueError:
if colid in ['row', '__row__']:
return self.row
elif colid in ['sheet', '__sheet__']:
return self.sheet
raise KeyError(colid)
class ColumnExpr(Column):
def __init__(self, name, expr=None):
super().__init__(name)
self.expr = expr or name
def calcValue(self, row):
return self.sheet.evalexpr(self.compiledExpr, row)
@property
def expr(self):
return self._expr
@expr.setter
def expr(self, expr):
self._expr = expr
self.compiledExpr = compile(expr, '', 'eval') if expr else None
###
def confirm(prompt):
yn = input(prompt, value='no', record=False)[:1]
if not yn or yn not in 'Yy':
fail('disconfirmed')
return True
import unicodedata
@functools.lru_cache(maxsize=8192)
def clipstr(s, dispw):
'''Return clipped string and width in terminal display characters.
Note: width may differ from len(s) if East Asian chars are 'fullwidth'.'''
w = 0
ret = ''
ambig_width = options.disp_ambig_width
for c in s:
if c != ' ' and unicodedata.category(c) in ('Cc', 'Zs', 'Zl'): # control char, space, line sep
c = options.disp_oddspace
if c:
c = c[0] # multi-char disp_oddspace just uses the first char
ret += c
eaw = unicodedata.east_asian_width(c)
if eaw == 'A': # ambiguous
w += ambig_width
elif eaw in 'WF': # wide/full
w += 2
elif not unicodedata.combining(c):
w += 1
if w > dispw-len(options.disp_truncator)+1:
ret = ret[:-2] + options.disp_truncator # replace final char with ellipsis
w += len(options.disp_truncator)
break
return ret, w
## text viewer and dir browser
# rowdef: (linenum, str)
class TextSheet(Sheet):
'Displays any iterable source, with linewrap if wrap set in init kwargs or options.'
rowtype = 'lines'
filetype = 'txt'
columns = [
ColumnItem('linenum', 0, type=int, width=0),
ColumnItem('text', 1),
]
def __init__(self, name, source, **kwargs):
super().__init__(name, source=source, **kwargs)
options.set('save_filetype', 'txt', self)
@asyncthread
def reload(self):
self.rows = []
winWidth = min(self.columns[1].width or 78, vd().windowWidth-2)
wrap = options.wrap
for startingLine, text in enumerate(self.source):
if wrap and text:
for i, L in enumerate(textwrap.wrap(str(text), width=winWidth)):
self.addRow([startingLine+i+1, L])
else:
self.addRow([startingLine+1, text])
TextSheet.addCommand('v', 'visibility', 'options.set("wrap", not options.wrap, sheet); reload(); status("text%s wrapped" % ("" if options.wrap else " NOT")); ')
### Curses helpers
def clipdraw(scr, y, x, s, attr, w=None, rtl=False):
'Draw string `s` at (y,x)-(y,x+w), clipping with ellipsis char. if rtl, draw inside (x-w, x). Returns width drawn (max of w).'
if not scr:
return 0
_, windowWidth = scr.getmaxyx()
dispw = 0
try:
if w is None:
w = windowWidth-1
w = min(w, (x-1) if rtl else (windowWidth-x-1))
if w <= 0: # no room anyway
return 0
# convert to string just before drawing
clipped, dispw = clipstr(str(s), w)
if rtl:
# clearing whole area (w) has negative display effects; clearing just dispw area is useless
# scr.addstr(y, x-dispw-1, disp_column_fill*dispw, attr)
scr.addstr(y, x-dispw-1, clipped, attr)
else:
scr.addstr(y, x, disp_column_fill*w, attr) # clear whole area before displaying
scr.addstr(y, x, clipped, attr)
except Exception as e:
pass
# raise type(e)('%s [clip_draw y=%s x=%s dispw=%s w=%s clippedlen=%s]' % (e, y, x, dispw, w, len(clipped))
# ).with_traceback(sys.exc_info()[2])
return dispw
# https://stackoverflow.com/questions/19833315/running-system-commands-in-python-using-curses-and-panel-and-come-back-to-previ
class SuspendCurses:
'Context Manager to temporarily leave curses mode'
def __enter__(self):
curses.endwin()
def __exit__(self, exc_type, exc_val, tb):
newscr = curses.initscr()
newscr.refresh()
curses.doupdate()
class EnableCursor:
def __enter__(self):
with suppress(curses.error):
curses.mousemask(0)
curses.curs_set(1)
def __exit__(self, exc_type, exc_val, tb):
with suppress(curses.error):
curses.curs_set(0)
curses.mousemask(-1)
def launchEditor(*args):
editor = os.environ.get('EDITOR') or fail('$EDITOR not set')
args = [editor] + list(args)
with SuspendCurses():
return subprocess.call(args)
def launchExternalEditor(v, linenum=0):
import tempfile
with tempfile.NamedTemporaryFile() as temp:
with open(temp.name, 'w') as fp:
fp.write(v)
if linenum:
launchEditor(temp.name, '+%s' % linenum)
else:
launchEditor(temp.name)
with open(temp.name, 'r') as fp:
r = fp.read()
if r[-1] == '\n': # trim inevitable trailing newline
r = r[:-1]
return r
def suspend():
import signal
with SuspendCurses():
os.kill(os.getpid(), signal.SIGSTOP)
# history: earliest entry first
def editText(scr, y, x, w, i=0, attr=curses.A_NORMAL, value='', fillchar=' ', truncchar='-', unprintablechar='.', completer=lambda text,idx: None, history=[], display=True):
'A better curses line editing widget.'
ESC='^['
ENTER='^J'
TAB='^I'
def until_get_wch():
'Ignores get_wch timeouts'
ret = None
while not ret:
try:
ret = scr.get_wch()
except curses.error:
pass
return ret
def splice(v, i, s):
'Insert `s` into string `v` at `i` (such that v[i] == s[0]).'
return v if i < 0 else v[:i] + s + v[i:]
def clean_printable(s):
'Escape unprintable characters.'
return ''.join(c if c.isprintable() else ('<%04X>' % ord(c)) for c in str(s))
def delchar(s, i, remove=1):
'Delete `remove` characters from str `s` beginning at position `i`.'
return s if i < 0 else s[:i] + s[i+remove:]
class CompleteState:
def __init__(self, completer_func):
self.comps_idx = -1
self.completer_func = completer_func
self.former_i = None
self.just_completed = False
def complete(self, v, i, state_incr):
self.just_completed = True
self.comps_idx += state_incr
if self.former_i is None:
self.former_i = i
try:
r = self.completer_func(v[:self.former_i], self.comps_idx)
except Exception as e:
# beep/flash; how to report exception?
return v, i
if not r:
# beep/flash to indicate no matches?
return v, i
v = r + v[i:]
return v, len(v)
def reset(self):
if self.just_completed:
self.just_completed = False
else:
self.former_i = None
self.comps_idx = -1
class HistoryState:
def __init__(self, history):
self.history = history
self.hist_idx = None
self.prev_val = None
def up(self, v, i):
if self.hist_idx is None:
self.hist_idx = len(self.history)
self.prev_val = v
if self.hist_idx > 0:
self.hist_idx -= 1
v = self.history[self.hist_idx]
i = len(v)
return v, i
def down(self, v, i):
if self.hist_idx is None:
return v, i
elif self.hist_idx < len(self.history)-1:
self.hist_idx += 1
v = self.history[self.hist_idx]
else:
v = self.prev_val
self.hist_idx = None
i = len(v)
return v, i
history_state = HistoryState(history)
complete_state = CompleteState(completer)
insert_mode = True
first_action = True
v = str(value) # value under edit
# i = 0 # index into v, initial value can be passed in as argument as of 1.2
if i != 0:
first_action = False
left_truncchar = right_truncchar = truncchar
def rfind_nonword(s, a, b):
if not s:
return 0
while not s[b].isalnum() and b >= a: # first skip non-word chars
b -= 1
while s[b].isalnum() and b >= a:
b -= 1
return b
while True:
if display:
dispval = clean_printable(v)
else:
dispval = '*' * len(v)
dispi = i # the onscreen offset within the field where v[i] is displayed
if len(dispval) < w: # entire value fits
dispval += fillchar*(w-len(dispval)-1)
elif i == len(dispval): # cursor after value (will append)
dispi = w-1
dispval = left_truncchar + dispval[len(dispval)-w+2:] + fillchar
elif i >= len(dispval)-w//2: # cursor within halfwidth of end
dispi = w-(len(dispval)-i)
dispval = left_truncchar + dispval[len(dispval)-w+1:]
elif i <= w//2: # cursor within halfwidth of beginning
dispval = dispval[:w-1] + right_truncchar
else:
dispi = w//2 # visual cursor stays right in the middle
k = 1 if w%2==0 else 0 # odd widths have one character more
dispval = left_truncchar + dispval[i-w//2+1:i+w//2-k] + right_truncchar
prew = clipdraw(scr, y, x, dispval[:dispi], attr, w)
clipdraw(scr, y, x+prew, dispval[dispi:], attr, w-prew+1)
scr.move(y, x+prew)
ch = vd().getkeystroke(scr)
if ch == '': continue
elif ch == 'KEY_IC': insert_mode = not insert_mode
elif ch == '^A' or ch == 'KEY_HOME': i = 0
elif ch == '^B' or ch == 'KEY_LEFT': i -= 1
elif ch in ('^C', '^Q', ESC): raise EscapeException(ch)
elif ch == '^D' or ch == 'KEY_DC': v = delchar(v, i)
elif ch == '^E' or ch == 'KEY_END': i = len(v)
elif ch == '^F' or ch == 'KEY_RIGHT': i += 1
elif ch in ('^H', 'KEY_BACKSPACE', '^?'): i -= 1; v = delchar(v, i)
elif ch == TAB: v, i = complete_state.complete(v, i, +1)
elif ch == 'KEY_BTAB': v, i = complete_state.complete(v, i, -1)
elif ch == ENTER: break
elif ch == '^K': v = v[:i] # ^Kill to end-of-line
elif ch == '^O': v = launchExternalEditor(v)
elif ch == '^R': v = str(value) # ^Reload initial value
elif ch == '^T': v = delchar(splice(v, i-2, v[i-1]), i) # swap chars
elif ch == '^U': v = v[i:]; i = 0 # clear to beginning
elif ch == '^V': v = splice(v, i, until_get_wch()); i += 1 # literal character
elif ch == '^W': j = rfind_nonword(v, 0, i-1); v = v[:j+1] + v[i:]; i = j+1 # erase word
elif ch == '^Z': suspend()
elif history and ch == 'KEY_UP': v, i = history_state.up(v, i)
elif history and ch == 'KEY_DOWN': v, i = history_state.down(v, i)
elif ch.startswith('KEY_'): pass
else:
if first_action:
v = ''
if insert_mode:
v = splice(v, i, ch)
else:
v = v[:i] + ch + v[i+1:]
i += 1
if i < 0: i = 0
if i > len(v): i = len(v)
first_action = False
complete_state.reset()
return v
class ColorMaker:
def __init__(self):
self.attrs = {}
self.color_attrs = {}
self.colorcache = {}
def setup(self):
if options.use_default_colors:
curses.use_default_colors()
default_bg = -1
else:
default_bg = curses.COLOR_BLACK
self.color_attrs['black'] = curses.color_pair(0)
for c in range(0, options.force_256_colors and 256 or curses.COLORS):
curses.init_pair(c+1, c, default_bg)
self.color_attrs[str(c)] = curses.color_pair(c+1)
for c in 'red green yellow blue magenta cyan white'.split():
colornum = getattr(curses, 'COLOR_' + c.upper())
self.color_attrs[c] = curses.color_pair(colornum+1)
for a in 'normal blink bold dim reverse standout underline'.split():
self.attrs[a] = getattr(curses, 'A_' + a.upper())
def keys(self):
return list(self.attrs.keys()) + list(self.color_attrs.keys())
def __getitem__(self, colornamestr):
return self._colornames_to_attr(colornamestr)
def __getattr__(self, optname):
'colors.color_foo returns colors[options.color_foo]'
return self.get_color(optname).attr
@functools.lru_cache()
def resolve_colors(self, colorstack):
'Returns the curses attribute for the colorstack, a list of color option names sorted highest-precedence color first.'
attr = CursesAttr()
for coloropt in colorstack:
c = self.get_color(coloropt)
attr = attr.update_attr(c)
return attr
def _colornames_to_attr(self, colornamestr, precedence=0):
attr = CursesAttr(0, precedence)
for colorname in colornamestr.split(' '):
if colorname in self.color_attrs:
if not attr.color:
attr.color = self.color_attrs[colorname.lower()]
elif colorname in self.attrs:
attr.attributes |= self.attrs[colorname.lower()]
return attr
def get_color(self, optname, precedence=0):
'colors.color_foo returns colors[options.color_foo]'
r = self.colorcache.get(optname, None)
if r is None:
coloropt = options._get(optname)
colornamestr = coloropt.value if coloropt else optname
r = self.colorcache[optname] = self._colornames_to_attr(colornamestr, precedence)
return r
colors = ColorMaker()
def setupcolors(stdscr, f, *args):
curses.raw() # get control keys instead of signals
curses.meta(1) # allow "8-bit chars"
curses.mousemask(-1) # even more than curses.ALL_MOUSE_EVENTS
curses.mouseinterval(0) # very snappy but does not allow for [multi]click
curses.mouseEvents = {}
for k in dir(curses):
if k.startswith('BUTTON') or k == 'REPORT_MOUSE_POSITION':
curses.mouseEvents[getattr(curses, k)] = k
return f(stdscr, *args)
def wrapper(f, *args):
return curses.wrapper(setupcolors, f, *args)
### external interface
def run(*sheetlist):
'Main entry point; launches vdtui with the given sheets already pushed (last one is visible)'
# reduce ESC timeout to 25ms. http://en.chys.info/2009/09/esdelay-ncurses/
os.putenv('ESCDELAY', '25')
ret = wrapper(cursesMain, sheetlist)
if ret:
print(ret)
def cursesMain(_scr, sheetlist):
'Populate VisiData object with sheets from a given list.'
colors.setup()
for vs in sheetlist:
vd().push(vs) # first push does a reload
status('Ctrl+H opens help')
return vd().run(_scr)
def loadConfigFile(fnrc, _globals=None):
p = Path(fnrc)
if p.exists():
try:
code = compile(open(p.resolve()).read(), p.resolve(), 'exec')
exec(code, _globals or globals())
except Exception as e:
exceptionCaught(e)
def addGlobals(g):
'importers can call `addGlobals(globals())` to have their globals accessible to execstrings'
globals().update(g)
def getGlobals():
return globals()
if __name__ == '__main__':
run(*(TextSheet('contents', open(src)) for src in sys.argv[1:]))
visidata-1.5.2/visidata/pivot.py 0000660 0001750 0001750 00000007373 13416500243 017340 0 ustar kefala kefala 0000000 0000000 from visidata import *
Sheet.addCommand('W', 'pivot', 'vd.push(SheetPivot(sheet, [cursorCol]))')
# rowdef: (tuple(keyvalues), dict(variable_value -> list(rows)))
class SheetPivot(Sheet):
'Summarize key columns in pivot table and display as new sheet.'
rowtype = 'aggregated rows'
def __init__(self, srcsheet, variableCols):
self.variableCols = variableCols
super().__init__(srcsheet.name+'_pivot_'+''.join(c.name for c in variableCols),
source=srcsheet)
def reload(self):
self.nonpivotKeyCols = []
for colnum, col in enumerate(self.source.keyCols):
if col not in self.variableCols:
newcol = Column(col.name, origcol=col, width=col.width, type=col.type,
getter=lambda col,row,colnum=colnum: row[0][colnum])
self.nonpivotKeyCols.append(newcol)
# two different threads for better interactive display
self.reloadCols()
self.reloadRows()
@asyncthread
def reloadCols(self):
self.columns = copy(self.nonpivotKeyCols)
self.setKeys(self.columns)
aggcols = [(c, aggregator) for c in self.source.visibleCols for aggregator in getattr(c, 'aggregators', [])]
if not aggcols:
aggcols = [(c, aggregators["count"]) for c in self.variableCols]
for col in self.variableCols:
for aggcol, aggregator in aggcols:
aggname = '%s_%s' % (aggcol.name, aggregator.__name__)
allValues = set()
for value in Progress(col.getValues(self.source.rows), 'pivoting', total=len(self.source.rows)):
if value not in allValues:
allValues.add(value)
c = Column('%s_%s' % (aggname, value),
type=aggregator.type or aggcol.type,
getter=lambda col,row,aggcol=aggcol,aggvalue=value,agg=aggregator: agg(aggcol, row[1].get(aggvalue, [])))
c.aggvalue = value
self.addColumn(c)
if aggregator.__name__ != 'count': # already have count above
c = Column('Total_' + aggname,
type=aggregator.type or aggcol.type,
getter=lambda col,row,aggcol=aggcol,agg=aggregator: agg(aggcol, sum(row[1].values(), [])))
self.addColumn(c)
c = Column('Total_count',
type=int,
getter=lambda col,row: len(sum(row[1].values(), [])))
self.addColumn(c)
@asyncthread
def reloadRows(self):
rowidx = {}
self.rows = []
for r in Progress(self.source.rows, 'pivoting'):
keys = tuple(forward(keycol.origcol.getTypedValue(r)) for keycol in self.nonpivotKeyCols)
formatted_keys = tuple(wrapply(c.format, v) for v, c in zip(keys, self.nonpivotKeyCols))
pivotrow = rowidx.get(formatted_keys)
if pivotrow is None:
pivotrow = (keys, {})
rowidx[formatted_keys] = pivotrow
self.addRow(pivotrow)
for col in self.variableCols:
varval = col.getTypedValueOrException(r)
matchingRows = pivotrow[1].get(varval)
if matchingRows is None:
pivotrow[1][varval] = [r]
else:
matchingRows.append(r)
SheetPivot.addCommand('z'+ENTER, 'dive-cell', 'vs=copy(source); vs.name+="_%s"%cursorCol.aggvalue; vs.rows=cursorRow[1].get(cursorCol.aggvalue, []); vd.push(vs)')
SheetPivot.addCommand(ENTER, 'dive-row', 'vs=copy(source); vs.name+="_%s"%"+".join(cursorRow[0]); vs.rows=sum(cursorRow[1].values(), []); vd.push(vs)')
visidata-1.5.2/visidata/join.py 0000660 0001750 0001750 00000017401 13416252050 017127 0 ustar kefala kefala 0000000 0000000 import collections
import itertools
import functools
from copy import copy
from visidata import asyncthread, Progress, status, fail, error
from visidata import ColumnItem, ColumnExpr, SubrowColumn, Sheet, Column
from visidata import SheetsSheet
SheetsSheet.addCommand('&', 'join-sheets', 'vd.replace(createJoinedSheet(selectedRows or fail("no sheets selected to join"), jointype=chooseOne(jointypes)))')
def createJoinedSheet(sheets, jointype=''):
sheets[1:] or error("join requires more than 1 sheet")
if jointype == 'append':
return SheetConcat('&'.join(vs.name for vs in sheets), sources=sheets)
elif jointype == 'extend':
vs = copy(sheets[0])
vs.name = '+'.join(vs.name for vs in sheets)
vs.reload = functools.partial(ExtendedSheet_reload, vs, sheets)
vs.rows = tuple() # to induce reload on first push, see vdtui
return vs
else:
return SheetJoin('+'.join(vs.name for vs in sheets), sources=sheets, jointype=jointype)
jointypes = {k:k for k in ["inner", "outer", "full", "diff", "append", "extend"]}
def joinkey(sheet, row):
return tuple(c.getDisplayValue(row) for c in sheet.keyCols)
def groupRowsByKey(sheets, rowsBySheetKey, rowsByKey):
with Progress(gerund='grouping', total=sum(len(vs.rows) for vs in sheets)*2) as prog:
for vs in sheets:
# tally rows by keys for each sheet
rowsBySheetKey[vs] = collections.defaultdict(list)
for r in vs.rows:
prog.addProgress(1)
key = joinkey(vs, r)
rowsBySheetKey[vs][key].append(r)
for vs in sheets:
for r in vs.rows:
prog.addProgress(1)
key = joinkey(vs, r)
if key not in rowsByKey: # gather for this key has not been done yet
# multiplicative for non-unique keys
rowsByKey[key] = []
for crow in itertools.product(*[rowsBySheetKey[vs2].get(key, [None]) for vs2 in sheets]):
rowsByKey[key].append([key] + list(crow))
#### slicing and dicing
# rowdef: [(key, ...), sheet1_row, sheet2_row, ...]
# if a sheet does not have this key, sheet#_row is None
class SheetJoin(Sheet):
'Column-wise join/merge. `jointype` constructor arg should be one of jointypes.'
@asyncthread
def reload(self):
sheets = self.sources
# first item in joined row is the key tuple from the first sheet.
# first columns are the key columns from the first sheet, using its row (0)
self.columns = []
for i, c in enumerate(sheets[0].keyCols):
self.addColumn(SubrowColumn(c.name, ColumnItem(c.name, i, type=c.type, width=c.width), 0))
self.setKeys(self.columns)
for sheetnum, vs in enumerate(sheets):
# subsequent elements are the rows from each source, in order of the source sheets
ctr = collections.Counter(c.name for c in vs.nonKeyVisibleCols)
for c in vs.nonKeyVisibleCols:
newname = c.name if ctr[c.name] == 1 else '%s_%s' % (vs.name, c.name)
self.addColumn(SubrowColumn(newname, c, sheetnum+1))
rowsBySheetKey = {}
rowsByKey = {}
groupRowsByKey(sheets, rowsBySheetKey, rowsByKey)
self.rows = []
with Progress(gerund='joining', total=len(rowsByKey)) as prog:
for k, combinedRows in rowsByKey.items():
prog.addProgress(1)
if self.jointype == 'full': # keep all rows from all sheets
for combinedRow in combinedRows:
self.addRow(combinedRow)
elif self.jointype == 'inner': # only rows with matching key on all sheets
for combinedRow in combinedRows:
if all(combinedRow):
self.addRow(combinedRow)
elif self.jointype == 'outer': # all rows from first sheet
for combinedRow in combinedRows:
if combinedRow[1]:
self.addRow(combinedRow)
elif self.jointype == 'diff': # only rows without matching key on all sheets
for combinedRow in combinedRows:
if not all(combinedRow):
self.addRow(combinedRow)
## for ExtendedSheet_reload below
class ExtendedColumn(Column):
def calcValue(self, row):
key = joinkey(self.sheet.joinSources[0], row)
srcsheet = self.sheet.joinSources[self.sheetnum]
srcrow = self.sheet.rowsBySheetKey[srcsheet][key]
if srcrow[0]:
return self.sourceCol.calcValue(srcrow[0])
@asyncthread
def ExtendedSheet_reload(self, sheets):
self.joinSources = sheets
# first item in joined row is the key tuple from the first sheet.
# first columns are the key columns from the first sheet, using its row (0)
self.columns = []
for i, c in enumerate(sheets[0].keyCols):
self.addColumn(copy(c))
self.setKeys(self.columns)
for i, c in enumerate(sheets[0].nonKeyVisibleCols):
self.addColumn(copy(c))
for sheetnum, vs in enumerate(sheets[1:]):
# subsequent elements are the rows from each source, in order of the source sheets
ctr = collections.Counter(c.name for c in vs.nonKeyVisibleCols)
for c in vs.nonKeyVisibleCols:
newname = '%s_%s' % (vs.name, c.name)
newcol = ExtendedColumn(newname, sheetnum=sheetnum+1, sourceCol=c)
self.addColumn(newcol)
self.rowsBySheetKey = {} # [srcSheet][key] -> list(rowobjs from sheets[0])
rowsByKey = {} # [key] -> [key, rows0, rows1, ...]
groupRowsByKey(sheets, self.rowsBySheetKey, rowsByKey)
self.rows = []
with Progress(gerund='joining', total=len(rowsByKey)) as prog:
for k, combinedRows in rowsByKey.items():
prog.addProgress(1)
for combinedRow in combinedRows:
if combinedRow[1]:
self.addRow(combinedRow[1])
## for SheetConcat
class ColumnConcat(Column):
def __init__(self, name, colsBySheet, **kwargs):
super().__init__(name, **kwargs)
self.colsBySheet = colsBySheet
def calcValue(self, row):
srcSheet, srcRow = row
srcCol = self.colsBySheet.get(srcSheet, None)
if srcCol:
return srcCol.calcValue(srcRow)
def setValue(self, row, v):
srcSheet, srcRow = row
srcCol = self.colsBySheet.get(srcSheet, None)
if srcCol:
srcCol.setValue(srcRow, v)
else:
fail('column not on source sheet')
# rowdef: (Sheet, row)
class SheetConcat(Sheet):
'combination of multiple sheets by row concatenation'
def reload(self):
self.rows = []
for sheet in self.sources:
for r in sheet.rows:
self.addRow((sheet, r))
self.columns = []
self.addColumn(ColumnItem('origin_sheet', 0))
allColumns = {}
for srcsheet in self.sources:
for srccol in srcsheet.visibleCols:
colsBySheet = allColumns.get(srccol.name, None)
if colsBySheet is None:
colsBySheet = {} # dict of [Sheet] -> Column
allColumns[srccol.name] = colsBySheet
if isinstance(srccol, ColumnExpr):
combinedCol = copy(srccol)
else:
combinedCol = ColumnConcat(srccol.name, colsBySheet, type=srccol.type)
self.addColumn(combinedCol)
if srcsheet in colsBySheet:
status('%s has multiple columns named "%s"' % (srcsheet.name, srccol.name))
colsBySheet[srcsheet] = srccol
self.recalc() # to set .sheet on columns, needed if this reload becomes async
visidata-1.5.2/visidata/commands.tsv 0000660 0001750 0001750 00000072716 13416252050 020167 0 ustar kefala kefala 0000000 0000000 sheet longname prefix key keystrokes jump selection modifies logged video menupath helpstr
global add-sheet A A new n n y 3 sheet-new open new blank sheet with N columns
global advance-replay ^I ^I n n n n meta-replay-step execute next row in replaying sheet
global cancel-all g ^C g^C n n n n meta-threads-cancel-all abort all secondary threads
global check-version n n n n check VisiData version against given version
global cmdlog D D meta n n n 3 meta-cmdlog open CommandLog
global columns-all g C gC meta n n y meta-columns-all open Columns Sheet with all columns from all sheets
global describe-all g I gI derived n n y data-describe-all open descriptive statistics for all visible columns on all sheets
global error-recent ^E ^E meta n n n errors-recent view traceback for most recent error
global errors-all g ^E g^E meta n n n info-errors-all view traceback for most recent errors
global exec-longname Space Space n meta-exec-cmd execute command by name
global exec-python g ^X g^X n n n y python-exec execute Python statement in the global scope
global forget-sheet Q Q quit n n n sheet-quit-remove quit current sheet and remove it from the cmdlog
global freq-rows z F zF y use n y data-aggregate-summary open one-line summary for all selected rows
global no-op KEY_RESIZE KEY_RESIZE n n n n no operation
global open-file o o y n n y sheet-open-path open input in VisiData
global pause-replay ^U ^U n n n n meta-replay-toggle pause/resume replay
global prev-sheet ^^ ^^ swap n n n view-go-sheet-swap jump to previous sheet (swap with current sheet)
global pyobj-expr ^X ^X y n n y python-eval-push evaluate Python expression and open result as Python object
global pyobj-expr-row z ^X z^X n n n y python-eval-status evaluate Python expression, in context of current row, and open result as Python object
global pyobj-sheet g ^Y g^Y meta n n y python-push-sheet-object open current sheet as Python object
global quit-all g q gq quit n n n sheet-quit-all quit all sheets (clean exit)
global quit-sheet q q quit n n y * sheet-quit-current quit current sheet
global redraw ^L ^L n n n n refresh screen
global reload-all g ^R g^R n n unmodifies y reload-all reload all selected sheets
global save-all g ^S g^S n n n y sheet-save-all save all sheets to given file or directory)
global save-cmdlog ^D ^D n n n n meta-cmdlog-save save CommandLog to new .vd file
global setdiff-sheet sheet-set-diff set this sheet as the diff sheet for all new sheets
global sheets S S static meta n n y meta-sheets open Sheets Sheet
global sheets-graveyard g S gS static meta n n y sheets-graveyard open Sheets Graveyard
global show-version ^V ^V n n n n info-version show version information on status line
global status n n n n show given message on status line
global statuses ^P ^P static meta n n n meta-status-history open Status History
global stop-replay ^K ^K n n n n meta-replay-cancel cancel current replay
global suspend ^Z ^Z suspend n n n suspend VisiData process
global sysopen-help ^H ^H external n n n view vd man page
global threads-all ^T ^T meta n n n meta-threads open Threads Sheet
global toggle-profile ^_ ^_ n n n n meta-threads-profile turn profiling on for main process
global visidata-dir g D gD static n n y meta-visidata-dir open .visidata directory
XmlSheet addcol-xmlattr z a za n n y y add column for xml attribute
XmlSheet dive-row Enter Enter y n n y load table referenced in current row into memory
XmlSheet visibility v v n n n y show only columns in current row attributes
ThreadsSheet cancel-thread ^C ^C n n n n cancel-thread abort thread at current row
ThreadsSheet profile-row Enter Enter y n n n sheet-profile push profile sheet for this action
SheetsSheet cancel-row z ^C z^C n n n n abort thread on current row
SheetsSheet cancel-rows gz ^C gz^C n use n n abort thread on selected rows
SheetsSheet columns-selected g C gC y y n y sheet-columns-selected merge the selected sheets with visible columns from all, keeping rows according to jointype
SheetsSheet describe-selected g I gI y y n y sheet-describe-selected open Describe Sheet with all columns from selected sheets
SheetsSheet join-sheets & & y y n y sheet-join merge the selected sheets with visible columns from all, keeping rows according to jointype
SheetsSheet open-row Enter Enter y n n y sheet-open-row open sheet referenced in current row
SheetsSheet open-rows g Enter gEnter y use n y sheet-open-rows push selected sheets to top of sheets stack
SheetsSheet reload-all g ^R g^R n n unmodifies y reload-all reload all selected sheets
SheetPivot dive-cell z Enter zEnter y n n y open-source-cell open sheet of source rows aggregated in current cell
SheetPivot dive-row Enter Enter y n n y open-source-row open sheet of source rows aggregated in current row
SheetObject dive-row Enter Enter y n n y python-dive-row dive further into Python object
SheetObject edit-cell e e n n y y edit contents of current cell
SheetObject hide-hidden z v zv n n n y hide methods and hidden properties
SheetObject show-hidden g v gv n n n y show methods and hidden properties
SheetObject visibility v v n n n y toggle show/hide for methods and hidden properties
SheetNamedTuple dive-row Enter Enter y n n y dive further into Python object
SheetH5Obj dive-metadata A A y n n y open metadata sheet for this object
SheetH5Obj dive-row Enter Enter y n n y load table referenced in current row into memory
SheetFreqTable dup-row Enter Enter y n n y open-cell-source open copy of source sheet with rows described in current cell
SheetFreqTable select-row s s n change n y filter-source-select-bin select these entries in source sheet
SheetFreqTable stoggle-row t t n change n y filter-source-toggle-bin toggle these entries in source sheet
SheetFreqTable unselect-row u u n change n y filter-source-unselect-bin unselect these entries in source sheet
SheetDict dive-row Enter Enter y n n y python-dive-row dive further into Python object
SheetDict edit-cell e n n y y modify-edit-cell edit contents of current cell
Sheet add-row a a n n y y 3 modify-add-row-blank insert a blank row
Sheet add-rows g a ga n n y y 3 modify-add-row-many add N blank rows
Sheet addcol-bulk gz a gza n n y y modify-add-column-manyblank add N empty columns
Sheet addcol-expr = = n n y y 1,3,5 modify-add-column-expr create new column from Python expression, with column names as variables
Sheet addcol-new z a za n n y y 3 modify-add-column-blank add an empty column
Sheet addcol-sh z ; z; n n y y create new column from bash expression, with $columnNames as variables
Sheet addcol-subst * * n n y y 5 modify-add-column-regex-transform add column derived from current column, replay regex with subst (may include \1 backrefs)
Sheet aggregate-col + + n n n y 5 column-aggregator-add add aggregator to current column
Sheet aggregate-numerics g + g+ n n n y add aggregator to all numeric columns
Sheet cache-col z ' z' n n n y column-cache-clear add/reset cache for current column
Sheet cache-cols gz ' gz' n n n y column-cache-clear-all add/reset cache for all visible columns
Sheet capture-col ; ; n n n y 1,5 modify-add-column-regex-capture add new column from capture groups of regex; requires example row
Sheet columns-sheet C C meta n n y 4,5 meta-columns-sheet open Columns Sheet
Sheet contract-col ) ) n n n y unexpand current column; restore original column and remove other columns at this level
Sheet copy-cell z y zy n n n y data-clipboard-copy-cell yank (copy) current cell to clipboard
Sheet copy-cells gz y gzy n use n y yank (copy) contents of current column for selected rows to clipboard
Sheet copy-row y y n n n y data-clipboard-copy-row yank (copy) current row to clipboard
Sheet copy-selected g y gy n use n y 4 data-clipboard-copy-selected yank (copy) selected rows to clipboard
Sheet delete-cell z d zd n n y y modify-clear-cell delete (cut) current cell and move it to clipboard
Sheet delete-cells gz d gzd n use y y modify-clear-column-selected delete (cut) contents of current column for selected rows and move them to clipboard
Sheet delete-row d d n n y y 1,2,5 modify-delete-row delete current row and move it to clipboard
Sheet delete-selected g d gd n use y y 5 modify-delete-selected delete selected rows and move them to clipboard
Sheet describe-sheet I I derived n n y data-describe open descriptive statistics for all visible columns
Sheet dive-cell z Enter zEnter derived n n y
Sheet dive-row Enter Enter derived n n y
Sheet dup-rows g " g" derived use n y sheet-duplicate-all open duplicate sheet with all rows
Sheet dup-rows-deep gz " gz" copy use n y sheet-duplicate-deepcopy open duplicate sheet with deepcopy of all rows
Sheet dup-selected " " derived use n y 2 sheet-duplicate-selected open duplicate sheet with only selected rows
Sheet dup-selected-deep z " z" copy use n y sheet-duplicate-deepcopy-selected open duplicate sheet with deepcopy of selected rows
Sheet edit-cell e e n n y y 3,4,5 modify-edit-cell edit contents of current cell
Sheet edit-cells g e ge n use y y 4,5 modify-edit-column-selected set contents of current column for selected rows to same input
Sheet error-cell z ^E z^E meta n n n view traceback for error in current cell
Sheet expand-col ( ( n n n y 5 expand_col expand current column of containers fully
Sheet expand-col-depth z ( z( n n n y expand_col_deep expand current column of containers to given depth (0=fully)
Sheet expand-cols g ( g( n n n y expand_vcols expand all visible columns of containers fully
Sheet expand-cols-depth gz ( gz( n n n y expand_vcols_deep expand all visible columns of containers to given depth (0=fully)
Sheet fill-nulls f f n n y y modify-fill-column fills null cells in current column with contents of non-null cells up the current column
Sheet freeze-col ' ' n n n y column-freeze add a frozen copy of current column with all cells evaluated
Sheet freeze-sheet g ' g' copy n n y sheet-freeze open a frozen copy of current sheet with all visible columns evaluated
Sheet freq-col F F derived n n y data-aggregate-column open Frequency Table grouped on current column
Sheet freq-keys g F gF derived n n y 2 data-aggregate-keys open Frequency Table grouped by all key columns on the source sheet
Sheet freq-rows F F derived n n y data-aggregate-summary open one-line summary for all selected rows
Sheet go-bottom g j gj n n n n move all the way to the bottom of sheet
Sheet go-col-number z c zc n n n n view-go-column-number move to the given column number
Sheet go-col-regex c c n n n n view-go-column-regex move to the next column with name matching regex
Sheet go-down j j n n n n * move down
Sheet go-end n n n n go to last row and last column
Sheet go-home n n n n go to first row and first column
Sheet go-left h h n n n n * move left
Sheet go-leftmost g h gh n n n n view-go-far-left move all the way to the left
Sheet go-mouse BUTTON1_PRESSED n n n n view-go-mouse-row
Sheet go-right l l n n n n * move right
Sheet go-rightmost g l gl n n n n view-go-far-right move all the way to the right of sheet
Sheet go-row-number z r zr n n n n view-go-row-number move to the given row number
Sheet go-top g k gk n n n n move all the way to the top of sheet
Sheet go-up k k n n n n * move up
Sheet help-commands z ^H z^H meta n n n meta-commands view sheet of commands and keybindings
Sheet hide-col - - n n n y 2 column-hide hide current column
Sheet key-col ! ! n n n y 2,3,5 column-key-toggle toggle current column as a key column
Sheet key-col-off z ! z! n n n y column-key-unset unset current column as a key column
Sheet key-col-on g ! g! n n n y set current column as a key column
Sheet melt M M derived n n y 1,5 data-melt open melted sheet (unpivot)
Sheet melt-regex g M gM derived n n y data-melt-regex open melted sheet (unpivot), factoring columns
Sheet next-null z > z> n n n n view-go-next-null move down the current column to the next null value
Sheet next-page ^F ^F n n n n scroll one page forward
Sheet next-search n n n n n n 2 view-find-forward-repeat move to next match from last search
Sheet next-selected } } n use n n 2 view-go-next-selected move down the current column to the next selected row
Sheet next-value > > n n n n view-go-next-value move down the current column to the next value
Sheet open-config g O gO y n n y open ~/.visidatarc as text sheet
Sheet options-global O O meta static n n n meta-options open global OptionsSheet
Sheet options-sheet z O zO meta static n n n meta-options open OptionsSheet for current SheetType
Sheet page-left n n n n move cursor one page left
Sheet page-right n n n n move cursor one page right
Sheet paste-after p p n n y y 5 data-clipboard-paste-after paste clipboard rows after current row
Sheet paste-before P P n n y y data-clipboard-paste-before paste clipboard rows before current row
Sheet paste-cell z p zp n n y y data-clipboard-paste-cell set contents of current cell to last clipboard value
Sheet paste-cell-before z P zP n n y y prepend to contents of current column for current row to last clipboard value
Sheet paste-cells gz y gzp n use y y set current column for selected rows to the items from last clipboard
Sheet paste-selected gz p gzp n change y y data-clipboard-paste-selected set cells of current column for selected rows to last clipboard value
Sheet pivot W W derived n n y 5 data-pivot Pivot the current column into a new sheet
Sheet plot-column . . canvas n n y 2,3 data-plot-column graph the current column vs key columns Numeric key column is on the x-axis, while categorical key columns determine color
Sheet plot-numerics g . g. canvas n n y data-plot-allnumeric open a graph of all visible numeric columns vs key column
Sheet prev-null z < z< n n n n view-go-prev-null move up the current column to the next null value
Sheet prev-page ^B ^B n n n n scroll one page backward
Sheet prev-search N N n n n n view-find-backward-repeat move to previous match from last search
Sheet prev-selected { { n use n n 2 view-go-prev-selected move up the current column to the previous selected row
Sheet prev-value < < n n n n view-go-prev-value move up the current column to the next value
Sheet pyobj-cell z ^Y z^Y meta n n y python-push-cell-object open current cell as Python object
Sheet pyobj-row ^Y ^Y meta n n y python-push-row-object open current row as Python object
Sheet random-rows R R derived n n y rows-select-random open duplicate sheet with a random population subset of # rows
Sheet reload-row z ^R z^R n n unmodifies y
Sheet reload-selected gz ^R gz^R n n unmodifies y
Sheet reload-sheet ^R ^R n n should confirm if modified; marks as unmodified y 4,5 sheet-reload reload current sheet
Sheet rename-col ^ ^ n n y y 3 column-name-input edit name of current column
Sheet rename-col-selected z ^ z^ n y y y column-name-cell set name of current column to combined contents of current cell in selected rows (or current row)
Sheet rename-cols-row g ^ g^ n n y y column-name-all-selected set names of all unnamed visible columns to contents of selected rows (or current row)
Sheet rename-cols-selected gz ^ gz^ n use y y column-name-selected set names of all visible columns to combined contents of selected rows (or current row)
Sheet resize-col z _ z_ n n n n column-width-input adjust current column width to given number
Sheet resize-col-half z - z- n n n n column-width-half reduce width of current column by half
Sheet resize-col-max _ _ n n n n 2 column-width-full adjust width of current column to full
Sheet resize-cols-max g _ g_ n n n n column-width-all-full adjust width of all visible columns to full
Sheet save-col z ^S z^S n n n y sheet-save-column save current column to filename in format determined by extension
Sheet save-sheet ^S ^S n n n y 2,4 sheet-save save current sheet to filename in format determined by extension (default .tsv)
Sheet scroll-down z j zj n n n n scroll one row down
Sheet scroll-down REPORT_MOUSE_POSITION n n n n view-scroll-down move scroll_incr backward
Sheet scroll-left z h zh n n n n scroll one column left
Sheet scroll-leftmost n n n n scroll sheet to leftmost column
Sheet scroll-middle z z zz n n n n scroll current row to center of screen
Sheet scroll-mouse BUTTON1_RELEASED n n n n view-scroll-mouse-row
Sheet scroll-right z l zl n n n n scroll one column right
Sheet scroll-rightmost n n n n scroll sheet to rightmost column
Sheet scroll-up z k zk n n n n scroll one row up
Sheet scroll-up BUTTON4_PRESSED n n n n view-scroll-up move scroll_incr forward
Sheet search-col / / n n n n 2 view-find-row-curcol-forward search for regex forwards in current column
Sheet search-cols g / g/ n n n n view-find-row-viscol-forward search for regex forwards over all visible columns
Sheet search-expr z / z/ n n n n search by Python expression forwards
Sheet search-keys r r n n n n view-go-row-regex move to the next row with key matching regex
Sheet searchr-col ? ? n n n n view-find-row-curcol-backward search for regex backwards in current column
Sheet searchr-cols g ? g? n n n n view-find-row-viscol-backward search for regex backwards over all visible columns
Sheet searchr-expr z ? z? n n n n info-manpage search by Python expression backwards
Sheet select-after gz s gzs n change n y rows-select-from-cursor select all rows from cursor to bottom
Sheet select-before z s zs n change n y rows-select-to-cursor select all rows from top to cursor
Sheet select-col-regex | | n change n y 4 rows-select-regex select rows matching regex in current column
Sheet select-cols-regex g | g| n change n y rows-select-regex-all select rows matching regex in any visible column
Sheet select-equal-cell , , n change n y 4,5 rows-select-like-cell select rows matching current cell in current column
Sheet select-equal-row g , g, n change n y rows-select-like-row select rows matching current row in all visible columns
Sheet select-expr z | z| n change n y rows-select-expr select rows with a Python expression
Sheet select-row s s n change n y 2,5 rows-select-current select current row
Sheet select-rows g s gs n change n y 5 rows-select-all select all rows
Sheet set-option n n y set option on current sheet
Sheet setcol-expr g = g= n use y y 3,5 modify-set-column-selected-expr set current column for selected rows to result of Python expression
Sheet setcol-range gz = gz= n use y y 3 modify-set-column-sequence set current column for selected rows to the items in result of Python sequence expression
Sheet setcol-subst g * g* n use y y 4 modify-add-column-regex-transform-foo regex/subst - modify selected rows replacing regex with subst, which may include backreferences) \1 etc) for current column
Sheet setcol-subst-all gz * gz* n use y y regex/subst - modify selected rows replacing regex with subst, which may include backreferences (\1 etc) for all visible columns
Sheet show-aggregate z + z+ n n n y 1 column-aggregator-show display result of aggregator over all values in current column
Sheet show-cursor ^G ^G n n n n info-sheet show cursor position and bounds of current sheet on status line
Sheet show-expr z = z= n n n y python-eval-row evaluate Python expression on current row and show result on status line
Sheet slide-bottom g J gJ n n n y slide-row-bottom slide current row to the bottom of sheet
Sheet slide-down J J n n n y slide-row-down slide current row down
Sheet slide-down-n z J zJ n n y y slide-row-down-n slide current row down n
Sheet slide-left H H n n n y slide-col-left slide current column left
Sheet slide-left-n z H zH n n y y slide-col-left-n slide current column left n
Sheet slide-leftmost g H gH n n n y 5 slide-col-leftmost slide current column all the way to the left of sheet
Sheet slide-right L L n n n y 5 slide-col-right slide current column right
Sheet slide-right-n z L zL n n y y slide-col-right-n slide current column right n
Sheet slide-rightmost g L gL n n n y slide-col-rightmost slide current column all the way to the right of sheet
Sheet slide-top g K gK n n n y slide-row-top slide current row the top of sheet
Sheet slide-up K K n n n y slide-row-up slide current row up
Sheet slide-up-n z K zK n n y y slide-row-up-n slide current row up n
Sheet sort-asc [ [ n n n y rows-sort-asc sort ascending by current column
Sheet sort-desc ] ] n n n y rows-sort-desc sort descending by current column
Sheet sort-keys-asc g [ g[ n n n y rows-sort-keys-asc sort ascending by all key columns
Sheet sort-keys-desc g ] g] n n n y rows-sort-keys-desc sort descending by all key columns
Sheet split-col : : n n n y 1 modify-add-column-regex-split add new columns from regex split; # columns determined by example row at cursor
Sheet stoggle-after gz t gzt n change n y rows-toggle-from-cursor toggle select rows from cursor to bottom
Sheet stoggle-before z t zt n change n y rows-toggle-to-cursor toggle select rows from top to cursor
Sheet stoggle-row t t n change n y rows-toggle-current toggle selection of current row
Sheet stoggle-rows g t gt n change n y rows-toggle-all toggle selection of all rows
Sheet syscopy-cell z Y zY n n n n data-clipboard-copy-system-cell yank (copy) current cell to system clipboard
Sheet syscopy-cells gz Y gzY n use n n data-clipboard-copy-system-cell yank (copy) cells in current column from selected rows to system clipboard
Sheet syscopy-row Y Y n n n n data-clipboard-copy-system-row yank (copy) current row to system clipboard
Sheet syscopy-selected g Y gY n use n n data-clipboard-copy-system-selected yank (copy) selected rows to system clipboard
Sheet sysopen-cell z ^O z^O external n y y
Sheet sysopen-row ^O ^O external n y y 1
Sheet transpose T T derived n n y data-transpose open new sheet with rows and columns transposed
Sheet type-any z ~ z~ n n n y column-type-any set type of current column to anytype
Sheet type-currency $ $ n n n y 1 column-type-currency set type of current column to currency
Sheet type-date @ @ n n n y column-type-date set type of current column to date
Sheet type-float % % n n n y 2,3 column-type-float set type of current column to float
Sheet type-int # # n n n y 2 column-type-int set type of current column to int
Sheet type-len z # z# n n n y set type of current column to len
Sheet type-string ~ ~ n n n y 1 column-type-str set type of current column to str
Sheet unhide-cols g v gv n n n y column-unhide-all unhide all columns
Sheet unselect-after gz u gzu n change n y rows-unselect-from-cursor unselect all rows from cursor to bottom
Sheet unselect-before z u zu n change n y rows-unselect-to-cursor unselect all rows from top to cursor
Sheet unselect-col-regex \ \ n change n y 5 rows-unselect-regex unselect rows matching regex in current column
Sheet unselect-cols-regex g \ g\ n change n y rows-unselect-regex-all unselect rows matching regex in any visible column
Sheet unselect-expr z \ z\ n change n y rows-unselect-expr unselect rows with a Python expression
Sheet unselect-row u u n change n y rows-unselect-current unselect current row
Sheet unselect-rows g u gu n change n y 2,5 rows-unselect-all unselect all rows
Sheet view-cell V V derived n ? y sheet-open-cell view contents of current cell in a new sheet
Sheet visibility v v n n n y
ProfileSheet dive-cell z Enter zEnter y n n y open ProfileSheet for caller referenced in current cell
ProfileSheet dive-row Enter Enter y n n y open ProfileSheet for calls referenced in current row
ProfileSheet save-profile z ^S z^S n n n y sheet-save-profile save profile
ProfileSheet sysopen-row ^O ^O external n n open current file at referenced row in external $EDITOR
Plotter redraw ^L ^L n n n n redraw all pixels on canvas
Plotter visibility v v n n n y toggle show_graph_labels
PNGSheet plot-sheet . . y n n y data-plot-png plot this png
MbtilesSheet dive-row Enter Enter y n n y load table referenced in current row into memory
MbtilesSheet plot-row . . y n n y plot tiles in current row
DirSheet delete-row n n y modify-delete-row
DirSheet delete-selected n use y modify-delete-selected
DirSheet open-row Enter Enter y n n y sheet-open-row open current file as a new sheet
DirSheet open-rows g Enter gEnter use n y sheet-open-rows open selected files as new sheets
DirSheet reload-row z ^R z^R n n unmodifies sheet-specific-apply-edits undo pending changes to current row
DirSheet reload-rows gz ^R gz^R n use unmodifies undo pending changes to selected rows
DirSheet save-row z ^S z^S n n n sheet-specific-apply-edits apply changes to current row
DirSheet save-sheet ^S ^S n n n sheet-specific-apply-edits apply all changes on all rows
DirSheet sysopen-row ^O ^O external n 1 edit-row-external open current file in external $EDITOR
DirSheet sysopen-rows g ^O g^O external use 1 edit-rows-external open selected files in external $EDITOR
DescribeSheet dup-cell z Enter zEnter y n n y open-cell-source open copy of source sheet with rows described in current cell
DescribeSheet freq-row ENTER y n n y data-aggregate-source-column open a Frequency Table sheet grouped on column referenced in current row
DescribeSheet select-cell z s zs n change y rows-select-source-cell select rows on source sheet which are being described in current cell
DescribeSheet unselect-cell z u zu n change n y rows-unselect-source-cell unselect rows on source sheet which are being described in current cell
CommandLog replay-all g x gx n n n n replay contents of entire CommandLog
CommandLog replay-row x n n n n replay command in current row
CommandLog save-macro z ^S z^S n y n n save macro
CommandLog stop-replay ^C n n n n abort replay
ColumnsSheet aggregate-cols g + g+ n use n y column-aggregate-add-all add aggregators to selected source columns
ColumnsSheet freq-col Enter Enter y n n y data-aggregate-source-column open a Frequency Table grouped on column referenced in current row
ColumnsSheet hide-selected g - g- n use n y column-hide-selected hide selected columns on source sheet
ColumnsSheet join-cols & & n use n y add column from concatenating selected source columns
ColumnsSheet key-off-selected gz ! gz! n use n y unset selected rows as key columns on source sheet
ColumnsSheet key-selected g ! g! n use n y toggle selected rows as key columns on source sheet
ColumnsSheet resize-source-rows n use n y column-source-width-max adjust widths of selected source columns
ColumnsSheet type-any-selected gz ~ gz~ n use n y column-type-any-selected set type of selected columns to anytype
ColumnsSheet type-currency-selected g $ g$ n use n y 1 column-type-currency-selected set type of selected columns to currency
ColumnsSheet type-date-selected g @ g@ n use n y column-type-date-selected set type of selected columns to date
ColumnsSheet type-float-selected g % g% n use n y 2,3 column-type-float-selected set type of selected columns to float
ColumnsSheet type-int-selected g # g# n use n y 2 column-type-int-selected set type of selected columns to int
ColumnsSheet type-len-selected gz # gz# n use n y column-type-int-selected set type of selected columns to len
ColumnsSheet type-string-selected g ~ g~ n use n y 1 column-type-str-selected set type of selected columns to str
Canvas delete-cursor d d n n source y delete rows on source sheet contained within canvas cursor
Canvas delete-visible g d gd n n source y delete rows on source sheet visible on screen
Canvas dive-cursor Enter Enter y n n y open sheet of source rows contained within canvas cursor
Canvas dive-visible g Enter gEnter y n n y open sheet of source rows visible on screen
Canvas end-cursor BUTTON1_RELEASED n n n n end cursor box with left mouse button release
Canvas end-move BUTTON3_RELEASED n n n n mark canvas anchor point
Canvas go-bottom n n n n
Canvas go-down n n n n
Canvas go-down-small n n n n
Canvas go-left n n n n
Canvas go-left-small n n n n
Canvas go-leftmost n n n n
Canvas go-right n n n n
Canvas go-right-small n n n n
Canvas go-rightmost n n n n
Canvas go-top n n n n
Canvas go-up n n n n
Canvas go-up-small n n n n
Canvas resize-cursor-doubleheight g K gK n n n n
Canvas resize-cursor-doublewide g L gL n n n n
Canvas resize-cursor-halfheight g J gJ n n n n
Canvas resize-cursor-halfwide g H gH n n n n
Canvas resize-cursor-shorter K K n n n n
Canvas resize-cursor-taller J J n n n n
Canvas resize-cursor-thinner H H n n n n
Canvas resize-cursor-wider L L n n n n
Canvas select-cursor s s n change n y select rows on source sheet contained within canvas cursor
Canvas select-visible g s gs n change n y select rows on source sheet visible on screen
Canvas set-aspect z _ z_ n n n n set aspect ratio
Canvas start-cursor BUTTON1_PRESSED n n n n start cursor box with left mouse button press
Canvas start-move BUTTON3_PRESSED n n n n mark grid point to move
Canvas stoggle-cursor t t n change n y toggle selection of rows on source sheet contained within canvas cursor
Canvas stoggle-visible g t gt n change n y toggle selection of rows on source sheet visible on screen
Canvas unselect-cursor u u n change n y unselect rows on source sheet contained within canvas cursor
Canvas unselect-visible g u gu n change n y unselect rows on source sheet visible on screen
Canvas zoom-all _ _ n n n n zoom to fit full extent
Canvas zoom-cursor zz n n n n set visible bounds to cursor
Canvas zoomin-cursor + + n n n n zoom into cursor center
Canvas zoomin-mouse BUTTON4_PRESSED n n n n zoom in with scroll wheel
Canvas zoomout-cursor - n n n n zoom out from cursor center
Canvas zoomout-mouse REPORT_MOUSE_POSITION n n n n zoom out with scroll wheel
BaseSheet cancel-sheet ^C ^C n n n n meta-threads-cancel-sheet abort all threads on current sheet
BaseSheet reload-sheet ^R ^R n n should confirm if modified; marks as unmodified y sheet-reload reload current sheet
BaseSheet rename-sheet n n y rename current sheet to input
visidata-1.5.2/visidata/movement.py 0000660 0001750 0001750 00000012442 13416252050 020022 0 ustar kefala kefala 0000000 0000000 import itertools
import re
from visidata import vd, VisiData, error, status, Sheet, Column, regex_flags, rotate_range, fail
vd.searchContext = {} # regex, columns, backward to kwargs from previous search
Sheet.addCommand('c', 'go-col-regex', 'sheet.cursorVisibleColIndex=nextColRegex(sheet, input("column name regex: ", type="regex-col", defaultLast=True))')
Sheet.addCommand('r', 'search-keys', 'tmp=cursorVisibleColIndex; vd.moveRegex(sheet, regex=input("row key regex: ", type="regex-row", defaultLast=True), columns=keyCols or [visibleCols[0]]); sheet.cursorVisibleColIndex=tmp')
Sheet.addCommand('zc', 'go-col-number', 'sheet.cursorVisibleColIndex = int(input("move to column number: "))')
Sheet.addCommand('zr', 'go-row-number', 'sheet.cursorRowIndex = int(input("move to row number: "))')
Sheet.addCommand('/', 'search-col', 'vd.moveRegex(sheet, regex=input("/", type="regex", defaultLast=True), columns="cursorCol", backward=False)'),
Sheet.addCommand('?', 'searchr-col', 'vd.moveRegex(sheet, regex=input("?", type="regex", defaultLast=True), columns="cursorCol", backward=True)'),
Sheet.addCommand('n', 'next-search', 'vd.moveRegex(sheet, reverse=False)'),
Sheet.addCommand('N', 'prev-search', 'vd.moveRegex(sheet, reverse=True)'),
Sheet.addCommand('g/', 'search-cols', 'vd.moveRegex(sheet, regex=input("g/", type="regex", defaultLast=True), backward=False, columns="visibleCols")'),
Sheet.addCommand('g?', 'searchr-cols', 'vd.moveRegex(sheet, regex=input("g?", type="regex", defaultLast=True), backward=True, columns="visibleCols")'),
Sheet.addCommand('<', 'prev-value', 'moveToNextRow(lambda row,sheet=sheet,col=cursorCol,val=cursorValue: col.getValue(row) != val, reverse=True) or status("no different value up this column")'),
Sheet.addCommand('>', 'next-value', 'moveToNextRow(lambda row,sheet=sheet,col=cursorCol,val=cursorValue: col.getValue(row) != val) or status("no different value down this column")'),
Sheet.addCommand('{', 'prev-selected', 'moveToNextRow(lambda row,sheet=sheet: sheet.isSelected(row), reverse=True) or status("no previous selected row")'),
Sheet.addCommand('}', 'next-selected', 'moveToNextRow(lambda row,sheet=sheet: sheet.isSelected(row)) or status("no next selected row")'),
Sheet.addCommand('z<', 'prev-null', 'moveToNextRow(lambda row,col=cursorCol,isnull=isNullFunc(): isnull(col.getValue(row)), reverse=True) or status("no null down this column")'),
Sheet.addCommand('z>', 'next-null', 'moveToNextRow(lambda row,col=cursorCol,isnull=isNullFunc(): isnull(col.getValue(row))) or status("no null down this column")'),
def moveToNextRow(vs, func, reverse=False):
'Move cursor to next (prev if reverse) row for which func returns True. Returns False if no row meets the criteria.'
rng = range(vs.cursorRowIndex-1, -1, -1) if reverse else range(vs.cursorRowIndex+1, vs.nRows)
for i in rng:
try:
if func(vs.rows[i]):
vs.cursorRowIndex = i
return True
except Exception:
pass
return False
Sheet.moveToNextRow = moveToNextRow
def nextColRegex(sheet, colregex):
'Go to first visible column after the cursor matching `colregex`.'
pivot = sheet.cursorVisibleColIndex
for i in itertools.chain(range(pivot+1, len(sheet.visibleCols)), range(0, pivot+1)):
c = sheet.visibleCols[i]
if re.search(colregex, c.name, regex_flags()):
return i
fail('no column name matches /%s/' % colregex)
def moveRegex(vd, sheet, *args, **kwargs):
list(vd.searchRegex(sheet, *args, moveCursor=True, **kwargs))
VisiData.moveRegex = moveRegex
# kwargs: regex=None, columns=None, backward=False
def searchRegex(vd, sheet, moveCursor=False, reverse=False, **kwargs):
'Set row index if moveCursor, otherwise return list of row indexes.'
def findMatchingColumn(sheet, row, columns, func):
'Find column for which func matches the displayed value in this row'
for c in columns:
if func(c.getDisplayValue(row)):
return c
vd.searchContext.update(kwargs)
regex = kwargs.get("regex")
if regex:
vd.searchContext["regex"] = re.compile(regex, regex_flags()) or error('invalid regex: %s' % regex)
regex = vd.searchContext.get("regex") or fail("no regex")
columns = vd.searchContext.get("columns")
if columns == "cursorCol":
columns = [sheet.cursorCol]
elif columns == "visibleCols":
columns = tuple(sheet.visibleCols)
elif isinstance(columns, Column):
columns = [columns]
if not columns:
error('bad columns')
searchBackward = vd.searchContext.get("backward")
if reverse:
searchBackward = not searchBackward
matchingRowIndexes = 0
for r in rotate_range(len(sheet.rows), sheet.cursorRowIndex, reverse=searchBackward):
c = findMatchingColumn(sheet, sheet.rows[r], columns, regex.search)
if c:
if moveCursor:
sheet.cursorRowIndex = r
sheet.cursorVisibleColIndex = sheet.visibleCols.index(c)
return
else:
matchingRowIndexes += 1
yield r
status('%s matches for /%s/' % (matchingRowIndexes, regex.pattern))
VisiData.searchRegex = searchRegex
visidata-1.5.2/visidata/asyncthread.py 0000660 0001750 0001750 00000004367 13416500243 020504 0 ustar kefala kefala 0000000 0000000 import ctypes
import threading
from .vdtui import *
option('min_memory_mb', 0, 'minimum memory to continue loading and async processing')
theme('color_working', 'green', 'color of system running smoothly')
BaseSheet.addCommand('^C', 'cancel-sheet', 'cancelThread(*sheet.currentThreads or fail("no active threads on this sheet"))')
globalCommand('g^C', 'cancel-all', 'liveThreads=list(t for vs in vd.sheets for t in vs.currentThreads); cancelThread(*liveThreads); status("canceled %s threads" % len(liveThreads))')
globalCommand('^T', 'threads-all', 'vd.push(vd.threadsSheet)')
SheetsSheet.addCommand('z^C', 'cancel-row', 'cancelThread(*cursorRow.currentThreads)')
SheetsSheet.addCommand('gz^C', 'cancel-rows', 'for vs in selectedRows: cancelThread(*vs.currentThreads)')
def cancelThread(*threads, exception=EscapeException):
'Raise exception on another thread.'
for t in threads:
ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(t.ident), ctypes.py_object(exception))
SheetsSheet.columns += [
ColumnAttr('threads', 'currentThreads', type=len),
]
# each row is an augmented threading.Thread object
class ThreadsSheet(Sheet):
rowtype = 'threads'
columns = [
ColumnAttr('name'),
Column('process_time', type=float, getter=lambda col,row: elapsed_s(row)),
ColumnAttr('profile'),
ColumnAttr('status'),
ColumnAttr('exception'),
]
def reload(self):
self.rows = vd().threads
ThreadsSheet.addCommand('^C', 'cancel-thread', 'cancelThread(cursorRow)')
def elapsed_s(t):
return (t.endTime or time.process_time())-t.startTime
def checkMemoryUsage(vs):
min_mem = options.min_memory_mb
threads = vd.unfinishedThreads
if not threads:
return None
ret = ''
attr = 'color_working'
if min_mem:
tot_m, used_m, free_m = map(int, os.popen('free --total --mega').readlines()[-1].split()[1:])
ret = '[%dMB] ' % free_m + ret
if free_m < min_mem:
attr = 'color_warning'
warning('%dMB free < %dMB minimum, stopping threads' % (free_m, min_mem))
cancelThread(*vd().unfinishedThreads)
curses.flash()
return ret, attr
vd().threadsSheet = ThreadsSheet('thread_history')
vd().addHook('rstatus', checkMemoryUsage)
visidata-1.5.2/visidata/shell.py 0000660 0001750 0001750 00000021101 13416500243 017267 0 ustar kefala kefala 0000000 0000000 import os
import stat
import pwd
import grp
import subprocess
import contextlib
from visidata import Column, Sheet, LazyMapRow, asynccache, exceptionCaught, DeferredSetColumn
from visidata import Path, ENTER, date, asyncthread, confirm, fail, error, FileExistsError
from visidata import CellColorizer, RowColorizer
Sheet.addCommand('z;', 'addcol-sh', 'cmd=input("sh$ ", type="sh"); addShellColumns(cmd, sheet)')
def open_dir(p):
return DirSheet(p.name, source=p)
def addShellColumns(cmd, sheet):
shellcol = ColumnShell(cmd, source=sheet, width=0)
for i, c in enumerate([
shellcol,
Column(cmd+'_stdout', srccol=shellcol, getter=lambda col,row: col.srccol.getValue(row)[0]),
Column(cmd+'_stderr', srccol=shellcol, getter=lambda col,row: col.srccol.getValue(row)[1]),
]):
sheet.addColumn(c, index=sheet.cursorColIndex+i+1)
class ColumnShell(Column):
def __init__(self, name, cmd=None, **kwargs):
super().__init__(name, **kwargs)
self.expr = cmd or name
@asynccache(lambda col,row: (col, id(row)))
def calcValue(self, row):
try:
import shlex
args = []
lmr = LazyMapRow(self.source, row)
for arg in shlex.split(self.expr):
if arg.startswith('$'):
args.append(str(lmr[arg[1:]]))
else:
args.append(arg)
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return p.communicate()
except Exception as e:
exceptionCaught(e)
class DirSheet(Sheet):
'Sheet displaying directory, using ENTER to open a particular file. Edited fields are applied to the filesystem.'
rowtype = 'files' # rowdef: Path
columns = [
DeferredSetColumn('directory',
getter=lambda col,row: row.parent.relpath(col.sheet.source.resolve()),
setter=lambda col,row,val: col.sheet.moveFile(row, val)),
DeferredSetColumn('filename',
getter=lambda col,row: row.name + row.ext,
setter=lambda col,row,val: col.sheet.renameFile(row, val)),
DeferredSetColumn('pathname', width=0,
getter=lambda col,row: row.resolve(),
setter=lambda col,row,val: os.rename(row.resolve(), val)),
Column('ext', getter=lambda col,row: row.is_dir() and '/' or row.suffix),
DeferredSetColumn('size', type=int,
getter=lambda col,row: row.stat().st_size,
setter=lambda col,row,val: os.truncate(row.resolve(), int(val))),
DeferredSetColumn('modtime', type=date,
getter=lambda col,row: row.stat().st_mtime,
setter=lambda col,row,val: os.utime(row.resolve(), times=((row.stat().st_atime, float(val))))),
DeferredSetColumn('owner', width=0,
getter=lambda col,row: pwd.getpwuid(row.stat().st_uid).pw_name,
setter=lambda col,row,val: os.chown(row.resolve(), pwd.getpwnam(val).pw_uid, -1)),
DeferredSetColumn('group', width=0,
getter=lambda col,row: grp.getgrgid(row.stat().st_gid).gr_name,
setter=lambda col,row,val: os.chown(row.resolve(), -1, grp.getgrnam(val).pw_gid)),
DeferredSetColumn('mode', width=0,
getter=lambda col,row: '{:o}'.format(row.stat().st_mode),
setter=lambda col,row,val: os.chmod(row.resolve(), int(val, 8))),
Column('filetype', width=0, cache=True, getter=lambda col,row: subprocess.Popen(['file', '--brief', row.resolve()], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].strip()),
]
colorizers = [
# CellColorizer(4, None, lambda s,c,r,v: s.colorOwner(s,c,r,v)),
CellColorizer(8, 'color_change_pending', lambda s,c,r,v: s.changed(c, r)),
RowColorizer(9, 'color_delete_pending', lambda s,c,r,v: r in s.toBeDeleted),
]
nKeys = 2
@staticmethod
def colorOwner(sheet, col, row, val):
ret = ''
if col.name == 'group':
mode = row.stat().st_mode
if mode & stat.S_IXGRP: ret = 'bold '
if mode & stat.S_IWGRP: return ret + 'green'
if mode & stat.S_IRGRP: return ret + 'yellow'
elif col.name == 'owner':
mode = row.stat().st_mode
if mode & stat.S_IXUSR: ret = 'bold '
if mode & stat.S_IWUSR: return ret + 'green'
if mode & stat.S_IRUSR: return ret + 'yellow'
def changed(self, col, row):
try:
return isinstance(col, DeferredSetColumn) and col.changed(row)
except Exception:
return False
def deleteFiles(self, rows):
for r in rows:
if r not in self.toBeDeleted:
self.toBeDeleted.append(r)
def moveFile(self, row, val):
fn = row.name + row.ext
newpath = os.path.join(val, fn)
if not newpath.startswith('/'):
newpath = os.path.join(self.source.resolve(), newpath)
parent = Path(newpath).parent
if parent.exists():
if not parent.is_dir():
error('destination %s not a directory' % parent)
else:
with contextlib.suppress(FileExistsError):
os.makedirs(parent.resolve())
os.rename(row.resolve(), newpath)
row.fqpn = newpath
self.restat(row)
def renameFile(self, row, val):
newpath = row.with_name(val)
os.rename(row.resolve(), newpath.resolve())
row.fqpn = newpath
self.restat(row)
def removeFile(self, path):
if path.is_dir():
os.rmdir(path.resolve())
else:
os.remove(path.resolve())
def undoMod(self, row):
for col in self.visibleCols:
if getattr(col, '_modifiedValues', None) and id(row) in col._modifiedValues:
del col._modifiedValues[id(row)]
if row in self.toBeDeleted:
self.toBeDeleted.remove(row)
self.restat(row)
def save(self, *rows):
changes = []
deletes = {}
for r in list(rows or self.rows): # copy list because elements may be removed
if r in self.toBeDeleted:
deletes[id(r)] = r
else:
for col in self.visibleCols:
if self.changed(col, r):
changes.append((col, r))
if not changes and not deletes:
fail('nothing to save')
cstr = ''
if changes:
cstr += 'change %d attributes' % len(changes)
if deletes:
if cstr: cstr += ' and '
cstr += 'delete %d files' % len(deletes)
confirm('really %s? ' % cstr)
self._commit(changes, deletes)
@asyncthread
def _commit(self, changes, deletes):
oldrows = self.rows
self.rows = []
for r in oldrows:
try:
if id(r) in deletes:
self.removeFile(r)
else:
self.rows.append(r)
except Exception as e:
exceptionCaught(e)
for col, row in changes:
try:
col.realsetter(col, row, col._modifiedValues[id(row)])
self.restat(r)
except Exception as e:
exceptionCaught(e)
@asyncthread
def reload(self):
self.toBeDeleted = []
self.rows = []
basepath = self.source.resolve()
for folder, subdirs, files in os.walk(basepath):
subfolder = folder[len(basepath)+1:]
if subfolder.startswith('.'): continue
for fn in files:
if fn.startswith('.'): continue
p = Path(os.path.join(folder, fn))
self.rows.append(p)
# sort by modtime initially
self.rows.sort(key=lambda row: row.stat().st_mtime, reverse=True)
def restat(self, row):
row.stat(force=True)
DirSheet.addCommand(ENTER, 'open-row', 'vd.push(openSource(cursorRow))')
DirSheet.addCommand('g'+ENTER, 'open-rows', 'for r in selectedRows: vd.push(openSource(r.resolve()))')
DirSheet.addCommand('^O', 'sysopen-row', 'launchEditor(cursorRow.resolve())')
DirSheet.addCommand('g^O', 'sysopen-rows', 'launchEditor(*(r.resolve() for r in selectedRows))')
DirSheet.addCommand('^S', 'save-sheet', 'save()')
DirSheet.addCommand('z^S', 'save-row', 'save(cursorRow)')
DirSheet.addCommand('z^R', 'reload-row', 'undoMod(cursorRow)')
DirSheet.addCommand('gz^R', 'reload-rows', 'for r in self.selectedRows: undoMod(r)')
DirSheet.addCommand(None, 'delete-row', 'if cursorRow not in toBeDeleted: toBeDeleted.append(cursorRow); cursorRowIndex += 1')
DirSheet.addCommand(None, 'delete-selected', 'deleteFiles(selectedRows)')
visidata-1.5.2/visidata/motd.py 0000660 0001750 0001750 00000000700 13416252050 017125 0 ustar kefala kefala 0000000 0000000 from visidata import *
from visidata import __version__
option('motd_url', 'https://visidata.org/motd-'+__version__, 'source of randomized startup messages')
earthdays = lambda n: n*24*60*60
@asyncthread
def domotd():
try:
if options.motd_url:
p = urlcache(options.motd_url, earthdays(1))
line = random.choice(list(p))
status(line.split('\t')[0], priority=-1)
except Exception:
pass
visidata-1.5.2/visidata/freqtbl.py 0000660 0001750 0001750 00000016001 13416252050 017622 0 ustar kefala kefala 0000000 0000000 import math
from visidata import *
Sheet.addCommand('F', 'freq-col', 'vd.push(SheetFreqTable(sheet, cursorCol))')
Sheet.addCommand('gF', 'freq-keys', 'vd.push(SheetFreqTable(sheet, *keyCols))')
globalCommand('zF', 'freq-rows', 'vd.push(SheetFreqTable(sheet, Column("Total", getter=lambda col,row: "Total")))')
theme('disp_histogram', '*', 'histogram element character')
option('disp_histolen', 50, 'width of histogram column')
#option('histogram_bins', 0, 'number of bins for histogram of numeric columns')
#option('histogram_even_interval', False, 'if histogram bins should have even distribution of rows')
ColumnsSheet.addCommand(ENTER, 'freq-row', 'vd.push(SheetFreqTable(source[0], cursorRow))')
def valueNames(vals):
return '-'.join(str(v) for v in vals)
# rowdef: (keys, source_rows)
class SheetFreqTable(Sheet):
'Generate frequency-table sheet on currently selected column.'
rowtype = 'bins'
def __init__(self, sheet, *columns):
fqcolname = '%s_%s_freq' % (sheet.name, '-'.join(col.name for col in columns))
super().__init__(fqcolname, source=sheet)
self.origCols = columns
self.largest = 100
self.columns = [
Column(c.name, type=c.type if c.type in typemap else anytype, width=c.width, fmtstr=c.fmtstr,
getter=lambda col,row,i=i: row[0][i],
setter=lambda col,row,v,i=i,origCol=c: setitem(row[0], i, v) and origCol.setValues(row[1], v))
for i, c in enumerate(self.origCols)
]
self.setKeys(self.columns) # origCols are now key columns
nkeys = len(self.keyCols)
self.columns.extend([
Column('count', type=int, getter=lambda col,row: len(row[1]), sql='COUNT(*)'),
Column('percent', type=float, getter=lambda col,row: len(row[1])*100/col.sheet.source.nRows, sql=''),
Column('histogram', type=str, getter=lambda col,row: options.disp_histogram*(options.disp_histolen*len(row[1])//col.sheet.largest), width=options.disp_histolen+2, sql=''),
])
aggregatedCols = [Column(aggregator.__name__+'_'+c.name,
type=aggregator.type or c.type,
getter=lambda col,row,origcol=c,aggr=aggregator: aggr(origcol, row[1]),
sql='%s(%s)' % (aggregator, c.name) )
for c in self.source.visibleCols
for aggregator in getattr(c, 'aggregators', [])
]
self.columns.extend(aggregatedCols)
if aggregatedCols: # hide percent/histogram if aggregations added
for c in self.columns[nkeys+1:nkeys+3]:
c.hide()
self.groupby = columns
self.orderby = [(self.columns[nkeys], -1)] # count desc
def selectRow(self, row):
self.source.select(row[1]) # select all entries in the bin on the source sheet
return super().selectRow(row) # then select the bin itself on this sheet
def unselectRow(self, row):
self.source.unselect(row[1])
return super().unselectRow(row)
def numericBinning(self):
nbins = options.histogram_bins or int(len(self.source.rows) ** (1./2))
origCol = self.origCols[0]
self.columns[0].type = str
# separate rows with errors at the column from those without errors
errorbin = []
allbin = []
for row in Progress(self.source.rows, 'binning'):
try:
v = origCol.getTypedValue(row)
allbin.append((v, row))
except Exception as e:
errorbin.append((e, row))
# find bin pivots from non-error values
binPivots = []
sortedValues = sorted(allbin, key=lambda x: x[0])
if options.histogram_even_interval:
binsize = len(sortedValues)/nbins
pivotIdx = 0
for i in range(math.ceil(len(sortedValues)/binsize)):
firstVal = sortedValues[int(pivotIdx)][0]
binPivots.append(firstVal)
pivotIdx += binsize
else:
minval, maxval = sortedValues[0][0], sortedValues[-1][0]
binWidth = (maxval - minval)/nbins
binPivots = list((minval + binWidth*i) for i in range(0, nbins))
binPivots.append(None)
# put rows into bins (as self.rows) based on values
binMinIdx = 0
binMin = 0
for binMax in binPivots[1:-1]:
binrows = []
for i, (v, row) in enumerate(sortedValues[binMinIdx:]):
if binMax != binPivots[-2] and v > binMax:
break
binrows.append(row)
binMaxDispVal = origCol.format(binMax)
binMinDispVal = origCol.format(binMin)
if binMinIdx == 0:
binName = '<=%s' % binMaxDispVal
elif binMax == binPivots[-2]:
binName = '>=%s' % binMinDispVal
else:
binName = '%s-%s' % (binMinDispVal, binMaxDispVal)
self.addRow((binName, binrows))
binMinIdx += i
binMin = binMax
if errorbin:
self.rows.insert(0, ('errors', errorbin))
ntallied = sum(len(x[1]) for x in self.rows)
assert ntallied == len(self.source.rows), (ntallied, len(self.source.rows))
def discreteBinning(self):
rowidx = {}
for r in Progress(self.source.rows, 'binning'):
keys = list(forward(c.getTypedValue(r)) for c in self.origCols)
# wrapply will pass-through a key-able TypedWrapper
formatted_keys = tuple(wrapply(c.format, c.getTypedValue(r)) for c in self.origCols)
histrow = rowidx.get(formatted_keys)
if histrow is None:
histrow = (keys, [])
rowidx[formatted_keys] = histrow
self.addRow(histrow)
histrow[1].append(r)
self.largest = max(self.largest, len(histrow[1]))
self.rows.sort(key=lambda r: len(r[1]), reverse=True) # sort by num reverse
@asyncthread
def reload(self):
'Generate histrow for each row and then reverse-sort by length.'
self.rows = []
# if len(self.origCols) == 1 and self.origCols[0].type in (int, float, currency):
# self.numericBinning()
# else:
self.discreteBinning()
# automatically add cache to all columns now that everything is binned
for c in self.nonKeyVisibleCols:
c._cachedValues = collections.OrderedDict()
SheetFreqTable.addCommand('t', 'stoggle-row', 'toggle([cursorRow]); cursorDown(1)')
SheetFreqTable.addCommand('s', 'select-row', 'select([cursorRow]); cursorDown(1)')
SheetFreqTable.addCommand('u', 'unselect-row', 'unselect([cursorRow]); cursorDown(1)')
SheetFreqTable.addCommand(ENTER, 'dup-row', 'vs = copy(source); vs.name += "_"+valueNames(cursorRow[0]); vs.rows=copy(cursorRow[1]); vd.push(vs)')
# Command('v', 'options.histogram_even_interval = not options.histogram_even_interval; reload()', 'toggle histogram_even_interval option')
visidata-1.5.2/visidata/diff.py 0000660 0001750 0001750 00000001660 13416252050 017100 0 ustar kefala kefala 0000000 0000000 from visidata import theme, globalCommand, Sheet, CellColorizer
theme('color_diff', 'red', 'color of values different from --diff source')
theme('color_diff_add', 'yellow', 'color of rows/columns added to --diff source')
globalCommand(None, 'setdiff-sheet', 'setDiffSheet(sheet)')
def makeDiffColorizer(othersheet):
def colorizeDiffs(sheet, col, row, cellval):
if not row or not col:
return None
vcolidx = sheet.visibleCols.index(col)
rowidx = sheet.rows.index(row)
if vcolidx < len(othersheet.visibleCols) and rowidx < len(othersheet.rows):
otherval = othersheet.visibleCols[vcolidx].getDisplayValue(othersheet.rows[rowidx])
if cellval.display != otherval:
return 'color_diff'
else:
return 'color_diff_add'
return colorizeDiffs
def setDiffSheet(vs):
Sheet.colorizers.append(CellColorizer(8, None, makeDiffColorizer(vs)))
visidata-1.5.2/visidata/man/ 0000770 0001750 0001750 00000000000 13416503554 016375 5 ustar kefala kefala 0000000 0000000 visidata-1.5.2/visidata/man/vd.1 0000770 0001750 0001750 00000106015 13416503076 017075 0 ustar kefala kefala 0000000 0000000 .Dd January 12, 2019
.Dt vd \&1 "Quick Reference Guide"
.Os Linux/MacOS
.
.\" Long option with arg: .Lo f filetype format
.\" Long flag: .Lo f filetype
.de Lo
.It Cm -\\$1 Ns , Cm --\\$2 Ns = Ns Ar \\$3
..
.de Lf
.It Cm -\\$1 Ns , Cm --\\$2
..
.Sh NAME
.
.Nm VisiData
.Nd a terminal utility for exploring and arranging tabular data
.
.Sh SYNOPSIS
.
.Nm vd
.Op Ar options
.Op Ar input No ...
.
.Nm vd
.Op Ar options
.Cm --play Ar cmdlog
.Op Cm -w Ar waitsecs
.Op Cm --batch
.Op Cm -o Ar output
.Op Ar field Ns Cm = Ns Ar value No ...
.
.Sh DESCRIPTION
.Nm VisiData No is a multipurpose tool built on the Sy vdtui No platform that can be used to explore, clean, edit, and restructure data.
Rows can be selected, filtered, and grouped; columns can be rearranged, transformed, and derived via regex or Python expressions; and workflows can be saved, documented, and replayed.
.
.Ss REPLAY MODE
.Bl -tag -width XXXXXXXXXXXXXXXXXXXXXX -compact
.Lo p play cmdlog
.No replay a saved Ar cmdlog No within the interface
.
.Lo w replay-wait seconds
.No wait Ar seconds No between commands
.
.Lf b batch
replay in batch mode (with no interface)
.
.Lo o output file
.No save final visible sheet to Ar file No as .tsv
.
.It Sy --replay-movement
.No toggle Sy --play No to move cursor cell-by-cell
.It Ar field Ns Cm = Ns Ar value
.No replace \&"{ Ns Ar field Ns }\&" in Ar cmdlog No contents with Ar value
.El
.
.Ss Commands During Replay
.Bl -tag -width XXXXXXXXXXXXXXXXXXX -compact -offset XXX
.It Sy ^U
pause/resume replay
.It Sy Tab
execute next row in replaying sheet
.It Sy ^K
cancel current replay
.El
.
.Ss GLOBAL COMMANDS
In most cases, commands that affect selected rows will affect all rows if no rows are selected.
.Pp
.Ss Keystrokes for the Cautious
.Bl -tag -width XXXXXXXXXXXXXXX -compact
.It Ic " ^H"
view this man page
.It Ic "z^H"
view sheet of commands and keybindings
.It Ic " ^Q"
abort program immediately
.It Ic " ^C"
cancel user input or abort all async threads on current sheet
.It Ic " q"
quit current sheet
.It Ic "gq"
quit all sheets (clean exit)
.El
.Ss "Cursor Movement"
.
.Bl -tag -width XXXXXXXXXXXXXXX -compact
.
.It Ic "Arrow PgUp Home"
move as expected
.It Ic " h j k l"
move left/down/up/right
.It Ic "gh gj gk gl"
move all the way to the left/bottom/top/right of sheet
.It Ic " G gg"
move all the way to the bottom/top of sheet
.It Ic "^B ^F"
scroll one page back/forward
.Pp
.It Ic "^^" No (Ctrl-^)
jump to previous sheet (swaps with current sheet)
.Pp
.It Ic " / ?" Ar regex
.No search for Ar regex No forward/backward in current column
.It Ic "g/ g?" Ar regex
.No search for Ar regex No forward/backward over all visible columns
.It Ic "z/ z?" Ar expr
.No search by Python Ar expr No forward/backward in current column (with column names as variables)
.It Ic " n N"
move to next/previous match from last search
.Pp
.It Ic " < >"
move up/down to next value in current column
.It Ic "z< z>"
move up/down to next null in current column
.It Ic " { }"
move up/down to next selected row
.
.El
.Pp
.Bl -tag -width XXXXXXXXXXXXXXX -compact
.Pp
.It Ic " c" Ar regex
.No move to next column with name matching Ar regex
.It Ic " r" Ar regex
.No move to next row with key matching Ar regex
.It Ic "zc zr" Ar number
.No move to column/row Ar number No (0-based)
.Pp
.It Ic " H J K L"
slide current row/column left/down/up/right
.It Ic "gH gJ gK gL"
slide current row/column all the way to the left/bottom/top/right of sheet
.Pp
.It Ic "zh zj zk zl"
scroll one left/down/up/right
.El
.
.Ss Column Manipulation
.
.Bl -tag -width XXXXXXXXXXXXXXX -compact
.
.It Ic " _" Ns " (underscore)"
adjust width of current column
.It Ic "g_"
adjust width of all visible columns
.It Ic "z_" Ar number
.No adjust width of current column to Ar number
.Pp
.It Ic " -" Ns " (hyphen)"
hide current column
.It Ic "z-" Ns
reduce width of current column by half
.It Ic "gv" Ns
unhide all columns
.Pp
.It Ic "! z!" Ns
toggle/unset current column as a key column
.It Ic "~ # % $ @ z#"
set type of current column to str/int/float/currency/date/len
.It Ic " ^"
edit name of current column
.It Ic " g^"
set names of all unnamed visible columns to contents of selected rows (or current row)
.It Ic " z^"
set name of current column to combined contents of current cell in selected rows (or current row)
.It Ic "gz^"
set name of all visible columns to combined contents of current column for selected rows (or current row)
.Pp
.It Ic " =" Ar expr
.No create new column from Python Ar expr Ns , with column names as variables
.It Ic " g=" Ar expr
.No set current column for selected rows to result of Python Ar expr
.It Ic "gz=" Ar expr
.No set current column for selected rows to the items in result of Python sequence Ar expr
.It Ic " z=" Ar expr
.No evaluate Python expression on current row and show result on status line
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact
.It Ic " '" Ns " (tick)"
add a frozen copy of current column with all cells evaluated
.It Ic "g'"
open a frozen copy of current sheet with all visible columns evaluated
.It Ic "z' gz'"
reset cache for current/all visible column(s)
.Pp
.It Ic " \&:" Ar regex
.No add new columns from Ar regex No split; number of columns determined by example row at cursor
.It Ic " \&;" Ar regex
.No add new columns from capture groups of Ar regex No (also requires example row)
.It Ic "z" Ns Ic "\&;" Ar expr
.No add new column from bash Ar expr Ns , with Sy $ Ns columnNames as variables
.It Ic " *" Ar regex Ns Sy / Ns Ar subst
.No add column derived from current column, replacing Ar regex No with Ar subst No (may include Sy \e1 No backrefs)
.It Ic "g* gz*" Ar regex Ns Sy / Ns Ar subst
.No modify selected rows in current/all visible column(s), replacing Ar regex No with Ar subst No (may include Sy \e1 No backrefs)
.Pp
.It Ic " ( g("
.No expand current/all visible column(s) of lists (e.g. Sy [3] Ns ) or dicts (e.g. Sy {3} Ns ) fully
.It Ic "z( gz(" Ar depth
.No expand current/all visible column(s) of lists (e.g. Sy [3] Ns ) or dicts (e.g. Sy {3} Ns ) to given Ar depth ( Ar 0 Ns = fully)
.It Ic " )"
unexpand current column; restore original column and remove other columns at this level
.El
.Ss Row Selection
.
.Bl -tag -width XXXXXXXXXXXXXXX -compact
.
.It Ic " s t u"
select/toggle/unselect current row
.It Ic " gs gt gu"
select/toggle/unselect all rows
.It Ic " zs zt zu"
select/toggle/unselect rows from top to cursor
.It Ic "gzs gzt gzu"
select/toggle/unselect rows from cursor to bottom
.It Ic " | \e\ " Ns Ar regex
.No select/unselect rows matching Ar regex No in current column
.It Ic "g| g\e\ " Ns Ar regex
.No select/unselect rows matching Ar regex No in any visible column
.It Ic "z| z\e\ " Ns Ar expr
.No select/unselect rows matching Python Ar expr No in any visible column
.It Ic " \&," Ns " (comma)"
select rows matching current cell in current column
.It Ic "g\&,"
select rows matching current row in all visible columns
.
.El
.
.
.Ss Row Sorting/Filtering
.
.Bl -tag -width XXXXXXXXXXXXXXX -compact
.
.It Ic " [ ]"
sort ascending/descending by current column
.It Ic "g[ g]"
sort ascending/descending by all key columns
.It Ic " \&""
open duplicate sheet with only selected rows
.It Ic "g\&""
open duplicate sheet with all rows
.It Ic "gz\&""
open duplicate sheet with deepcopy of selected rows
.El
.Ss Editing Rows and Cells
.
.Bl -tag -width XXXXXXXXXXXXXXX -compact
.It Ic " a za"
append a blank row/column
.It Ic " ga gza" Ar number
.No append Ar number No blank rows/columns
.It Ic " d gd"
delete (cut) current/selected row(s) and move to clipboard
.It Ic " y gy"
yank (copy) current/all selected row(s) to clipboard
.It Ic " zy gzy"
yank (copy) contents of current column for current/selected row(s) to clipboard
.It Ic " zd gzd"
delete (cut) contents of current column for current/selected row(s) and move to clipboard
.It Ic " p P"
paste clipboard rows after/before current row
.It Ic " zp gzp"
set contents of current column for current/selected row(s) to last clipboard value
.It Ic " Y gY"
.No yank (copy) current/all selected row(s) to system clipboard (using Sy options.clipboard_copy_cmd Ns )
.It Ic " zY gzY"
.No yank (copy) contents of current column for current/selected row(s) to system clipboard (using Sy options.clipboard_copy_cmd Ns )
.It Ic " f"
fill null cells in current column with contents of non-null cells up the current column
.
.
.It Ic " e" Ar text
edit contents of current cell
.It Ic " ge" Ar text
.No set contents of current column for selected rows to Ar text
.
.El
.
.Ss " Commands While Editing Input"
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic "Enter ^C"
accept/abort input
.It Ic ^O
open external $EDITOR to edit contents
.It Ic ^R
reload initial value
.It Ic "^A ^E"
move to beginning/end of line
.It Ic "^B ^F"
move back/forward one character
.It Ic "^H ^D"
delete previous/current character
.It Ic ^T
transpose previous and current characters
.It Ic "^U ^K"
clear from cursor to beginning/end of line
.It Ic "Backspace Del"
delete previous/current character
.It Ic Insert
toggle insert mode
.It Ic "Up Down"
set contents to previous/next in history
.It Ic "Tab Shift+Tab"
autocomplete input (when available)
.
.El
.
.Ss Data Toolkit
.Bl -tag -width XXXXXXXXXXXXXXX -compact
.It Ic " o" Ar input
open
.Ar input No in Sy VisiData
.It Ic "^S g^S" Ar filename
.No save current/all sheet(s) to Ar filename No in format determined by extension (default .tsv)
.It ""
.No Note: if the format does not support multisave, or the Ar filename No ends in a Sy / Ns , a directory will be created.
.It Ic "z^S" Ar filename
.No save key columns and current column only to Ar filename No in format determined by extension (default .tsv)
.It Ic "^D" Ar filename.vd
.No save Sy CommandLog No to Ar filename.vd No file
.It Ic "A" Ar number
.No open new blank sheet with Ar number No columns
.It Ic "R" Ar number
pushes sheet with random population subset of
.Ar number No rows
.It Ic "T"
open new sheet with rows and columns transposed
.Pp
.It Ic " +" Ar aggregator
.No add Ar aggregator No to current column (see Sy "Frequency Table" Ns )
.It Ic "z+" Ar aggregator
.No display result of Ar aggregator No over values in selected rows for current column
.Pp
.El
.Ss Data Visualization
.Bl -tag -width XXXXXXXXXXXXX -compact
.It Ic " ." No (dot)
.No plot current numeric column vs key columns. The numeric key column is used for the x-axis; categorical key column values determine color.
.It Ic "g."
.No plot a graph of all visible numeric columns vs key columns.
.Pp
.El
.No If rows on the current sheet represent plottable coordinates (as in .shp or vector .mbtiles sources),
.Ic " ." No plots the current row, and Ic "g." No plots all selected rows (or all rows if none selected).
.Ss " Canvas-specific Commands"
.Bl -tag -width XXXXXXXXXXXXXXXXXX -compact -offset XXX
.It Ic " + -"
increase/decrease zoom level, centered on cursor
.It Ic " _" No (underscore)
zoom to fit full extent
.It Ic " s t u"
select/toggle/unselect rows on source sheet contained within canvas cursor
.It Ic "gs gt gu"
select/toggle/unselect rows on source sheet visible on screen
.It Ic " d"
delete rows on source sheet contained within canvas cursor
.It Ic "gd"
delete rows on source sheet visible on screen
.It Ic " Enter"
open sheet of source rows contained within canvas cursor
.It Ic "gEnter"
open sheet of source rows visible on screen
.It Ic " 1" No - Ic "9"
toggle display of layers
.It Ic "^L"
redraw all pixels on canvas
.It Ic " v"
.No toggle Ic show_graph_labels No option
.It Ic "mouse scrollwheel"
zoom in/out of canvas
.It Ic "left click-drag"
set canvas cursor
.It Ic "right click-drag"
scroll canvas
.El
.Ss Other Commands
.
.Bl -tag -width XXXXXXXXXXXXXXX -compact
.It Ic "Q"
.No quit current sheet and remove it from the Sy CommandLog
.It Ic "V"
view contents of current cell in a new TextSheet
.It Ic " v"
toggle sheet-specific visibility (text wrap on TextSheet, legends/axes on Graph)
.Pp
.It Ic "Space" Ar longname
.No execute command by its Ar longname
.Pp
.It Ic " ^E"
view traceback for most recent error
.It Ic "g^E"
view traceback for most recent errors
.It Ic "z^E"
view traceback for error in current cell
.Pp
.It Ic " ^L"
refresh screen
.It Ic " ^R"
reload current sheet
.It Ic "z^R"
clear cache for current column
.It Ic " ^Z"
suspend VisiData process
.It Ic " ^G"
show cursor position and bounds of current sheet on status line
.It Ic " ^V"
show version and copyright information on status line
.It Ic " ^P"
.No open Sy Status History
.
.El
.Pp
.Bl -tag -width XXXXXXXXXXXXXXX -compact
.It Ic " ^Y z^Y g^Y"
open current row/cell/sheet as Python object
.It Ic " ^X" Ar expr
.No evaluate Python Ar expr No and opens result as Python object
.It Ic "z^X" Ar expr
.No evaluate Python Ar expr No on current row and shows result on status line
.It Ic "g^X" Ar stmt
.No execute Python Ar stmt No in the global scope
.El
.
.Ss Internal Sheets List
.Bl -tag -width Xx -compact
.It Sy " \&."
.Sy Directory Sheet No " browse and modify properties of files in a directory"
.It " "
.It Sy Metasheets
.It Sy " \&."
.Sy Columns Sheet No (Shift+C) " edit column properties"
.It Sy " \&."
.Sy Sheets Sheet No (Shift+S) " jump between sheets or join them together"
.It Sy " \&."
.Sy Options Sheet No (Shift+O) " edit configuration options"
.It Sy " \&."
.Sy Commandlog No (Shift+D) " modify and save commands for replay"
.It Sy " \&."
.Sy Error Sheet No (^E) " view last error"
.It Sy " \&."
.Sy Status History No (^P) " view history of status messages"
.It Sy " \&."
.Sy Threads Sheet No (^T) " view, cancel, and profile asynchronous threads"
.Pp
.It Sy Derived Sheets
.It Sy " \&."
.Sy Frequency Table No (Shift+F) " group rows by column value, with aggregations of other columns"
.It Sy " \&."
.Sy Describe Sheet No (Shift+I) " view summary statistics for each column"
.It Sy " \&."
.Sy Pivot Table No (Shift+W) " group rows by key and summarize current column"
.It Sy " \&."
.Sy Melted Sheet No (Shift+M) " unpivot non-key columns into variable/value columns"
.El
.
.Ss INTERNAL SHEETS
.Ss Directory Sheet
.Bl -inset -compact
.It (sheet-specific commands)
.It Modifying any cell changes the in-memory value. Changes are only applied to the filesystem with Ic ^S
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic "Enter gEnter"
open current/selected file(s) as new sheet(s)
.It Ic " ^O g^O"
open current/selected file(s) in external $EDITOR
.It Ic " d gd"
schedule current/selected file(s) for deletion
.It Ic " ^R z^R gz^R"
reload information for all/current/selected file(s), undoing any pending changes
.It Ic "z^S ^S"
apply all deferred changes to current/all file(s)
.El
.
.Ss METASHEETS
.Ss Columns Sheet (Shift+C)
.Bl -inset -compact
.It Properties of columns on the source sheet can be changed with standard editing commands ( Ns Sy e ge g= Del Ns ) on the Sy Columns Sheet Ns . Multiple aggregators can be set by listing them (separated by spaces) in the aggregators column. The 'g' commands affect the selected rows, which are the literal columns on the source sheet.
.El
.Bl -inset -compact
.It (global commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic gC
.No open Sy Columns Sheet No for all visible columns on all sheets
.El
.Bl -inset -compact
.It (sheet-specific commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic " &"
add column from concatenating selected source columns
.It Ic "g! gz!"
toggle/unset selected columns as key columns on source sheet
.It Ic "g+" Ar aggregator
add Ar aggregator No to selected source columns
.It Ic "g-" No (hyphen)
hide selected columns on source sheet
.It Ic "g~ g# g% g$ g@ gz#"
set type of selected columns on source sheet to str/int/float/currency/date/len
.It Ic " Enter"
.No open a Sy Frequency Table No sheet grouped by column referenced in current row
.El
.
.Ss Sheets Sheet (Shift+S)
.Bl -inset -compact
.It (global commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic gS
.No open Sy Sheets Graveyard No which includes references to closed sheets
.El
.Bl -inset -compact
.It (sheet-specific commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic " Enter"
jump to sheet referenced in current row
.It Ic " a"
add row to reference a new blank sheet
.It Ic "gC"
.No open Sy Columns Sheet No with all columns from selected sheets
.It Ic "gI"
.No open Sy Describe Sheet No with all columns from selected sheets
.It Ic "g^R"
.No reload all selected sheets
.It Ic "z^C gz^C"
abort async threads for current/selected sheets(s)
.It Ic " &" Ar jointype
.No merge selected sheets with visible columns from all, keeping rows according to Ar jointype Ns :
.El
.Bl -tag -width x -compact -offset XXXXXXXXXXXXXXXXXXXX
.It Sy "\&."
.Sy inner No " keep only rows which match keys on all sheets"
.It Sy "\&."
.Sy outer No " keep all rows from first selected sheet"
.It Sy "\&."
.Sy full No " keep all rows from all sheets (union)"
.It Sy "\&."
.Sy diff No " keep only rows NOT in all sheets"
.It Sy "\&."
.Sy append No "keep all rows from all sheets (concatenation)"
.It Sy "\&."
.Sy extend No "copy first selected sheet, keeping all rows and sheet type, and extend with columns from other sheets"
.El
.
.Ss Options Sheet (Shift+O)
.Bl -inset -compact
.It (global commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic Shift+O
.No edit global options (apply to Sy all sheets Ns )
.It Ic zO
.No edit sheet options (apply to Sy this sheet No only)
.It Ic gO
.No open Sy ~/.visidatarc
.El
.Bl -inset -compact
.It (sheet-specific commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic "Enter e"
edit option at current row
.El
.
.Ss CommandLog (Shift+D)
.Bl -inset -compact
.It (global commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic gD
.No open Sy Directory Sheet No for Sy options.visidata_dir No (default: Sy ~/.visidata/ Ns ), which contains saved commandlogs and macros
.El
.Bl -inset -compact
.It (sheet-specific commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic " x"
replay command in current row
.It Ic " gx"
replay contents of entire CommandLog
.It Ic " ^C"
abort replay
.It Ic "z^S" Ar keystroke
.No save selected rows to macro mapped to Ar keystroke
.It ""
.No Macros are saved to Sy .visidata/macro/command-longname.vd Ns . The list of macros is saved at Sy .visidata/macros.vd No (keystroke, filename).
.El
.
.Ss DERIVED SHEETS
.Ss Frequency Table (Shift+F)
.Bl -inset -compact
.It A Sy Frequency Table No groups rows by one or more columns, and includes summary columns for those with aggregators.
.It (global commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic gF
open Frequency Table, grouped by all key columns on source sheet
.It Ic zF
open one-line summary for selected rows
.El
.Bl -inset -compact
.It (sheet-specific commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic " s t u"
select/toggle/unselect these entries in source sheet
.It Ic " Enter"
open sheet of source rows that are grouped in current cell
.El
.
.Ss Describe Sheet (Shift+I)
.Bl -inset -compact
.It (global commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic gI
.No open Sy Describe Sheet No for all visible columns on all sheets
.El
.Bl -inset -compact
.It (sheet-specific commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic "zs zu"
select/unselect rows on source sheet that are being described in current cell
.It Ic " !"
toggle/unset current column as a key column on source sheet
.It Ic " Enter"
.No open a Sy Frequency Table No sheet grouped on column referenced in current row
.It Ic "zEnter"
open copy of source sheet with rows described in current cell
.El
.
.Ss Pivot Table (Shift+W)
.Bl -inset -compact
.It Set key column(s) and aggregators on column(s) before pressing Sy Shift+W No on the column to pivot.
.It (sheet-specific commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic " Enter"
open sheet of source rows aggregated in current pivot row
.It Ic "zEnter"
open sheet of source rows aggregated in current pivot cell
.El
.Ss Melted Sheet (Shift+M)
.Bl -inset -compact
.It Open melted sheet (unpivot), with key columns retained and all non-key columns reduced to Variable-Value rows.
.It (global commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic "gM" Ar regex
.No open melted sheet (unpivot), with key columns retained and Ar regex No capture groups determining how the non-key columns will be reduced to Variable-Value rows.
.El
.Ss Python Object Sheet (^X ^Y g^Y z^Y)
.Bl -inset -compact
.It (sheet-specific commands)
.El
.Bl -tag -width XXXXXXXXXXXXXXX -compact -offset XXX
.It Ic " Enter"
dive further into Python object
.It Ic " e"
edit contents of current cell
.It Ic " v"
toggle show/hide for methods and hidden properties
.It Ic "gv zv"
show/hide methods and hidden properties
.El
.
.Sh COMMANDLINE OPTIONS
.Bl -tag -width XXXXXXXXXXXXXXXXXXXXXXXXXXX -compact
.
.Lo f filetype filetype
.No "tsv "
set loader to use for
.Ar filetype
instead of file extension
.
.Lo y confirm-overwrite F
.No "True "
overwrite existing files without confirmation
.
.It Cm --diff Ns = Ns Ar base
.No "None "
.No add colorizer for all sheets against Ar base
.
.El
.Bl -tag -width XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -compact
.It Sy --encoding Ns = Ns Ar "str " No "utf-8"
encoding passed to codecs.open
.It Sy --encoding-errors Ns = Ns Ar "str " No "surrogateescape"
encoding_errors passed to codecs.open
.It Sy --regex-flags Ns = Ns Ar "str " No "I"
flags to pass to re.compile() [AILMSUX]
.It Sy --default-width Ns = Ns Ar "int " No "20"
default column width
.It Sy --wrap No " False"
wrap text to fit window width on TextSheet
.It Sy --bulk-select-clear No " False"
clear selected rows before new bulk selections
.It Sy --cmd-after-edit Ns = Ns Ar "str " No "go-down"
command longname to execute after successful edit
.It Sy --col-cache-size Ns = Ns Ar "int " No "0"
max number of cache entries in each cached column
.It Sy --quitguard No " False"
confirm before quitting last sheet
.It Sy --null-value Ns = Ns Ar "NoneType " No "None"
a value to be counted as null
.It Sy --force-valid-colnames No " False"
clean column names to be valid Python identifiers
.It Sy --debug No " False"
exit on error and display stacktrace
.It Sy --curses-timeout Ns = Ns Ar "int " No "100"
curses timeout in ms
.It Sy --force-256-colors No " False"
use 256 colors even if curses reports fewer
.It Sy --use-default-colors No " False"
curses use default terminal colors
.It Sy --note-pending Ns = Ns Ar "str " No "\[u231B]"
note to display for pending cells
.It Sy --note-format-exc Ns = Ns Ar "str " No "?"
cell note for an exception during formatting
.It Sy --note-getter-exc Ns = Ns Ar "str " No "!"
cell note for an exception during computation
.It Sy --note-type-exc Ns = Ns Ar "str " No "!"
cell note for an exception during type conversion
.It Sy --note-unknown-type Ns = Ns Ar "str " No ""
cell note for unknown types in anytype column
.It Sy --scroll-incr Ns = Ns Ar "int " No "3"
amount to scroll with scrollwheel
.It Sy --skip Ns = Ns Ar "int " No "0"
skip first N lines of text input
.It Sy --confirm-overwrite Ns = Ns Ar "bool " No "True"
whether to prompt for overwrite confirmation on save
.It Sy --safe-error Ns = Ns Ar "str " No "#ERR"
error string to use while saving
.It Sy --header Ns = Ns Ar "int " No "1"
parse first N rows of certain formats as column names
.It Sy --delimiter Ns = Ns Ar "str " No " "
delimiter to use for tsv filetype
.It Sy --filetype Ns = Ns Ar "str " No ""
specify file type
.It Sy --save-filetype Ns = Ns Ar "str " No "tsv"
specify default file type to save as
.It Sy --tsv-safe-newline Ns = Ns Ar "str " No " "
replacement for tab character when saving to tsv
.It Sy --tsv-safe-tab Ns = Ns Ar "str " No " "
replacement for newline character when saving to tsv
.It Sy --clipboard-copy-cmd Ns = Ns Ar "str " No ""
command to copy stdin to system clipboard
.It Sy --visibility Ns = Ns Ar "int " No "0"
visibility level (0=low, 1=high)
.It Sy --min-memory-mb Ns = Ns Ar "int " No "0"
minimum memory to continue loading and async processing
.It Sy --replay-wait Ns = Ns Ar "float " No "0.0"
time to wait between replayed commands, in seconds
.It Sy --replay-movement No " False"
insert movements during replay
.It Sy --visidata-dir Ns = Ns Ar "str " No "~/.visidata/"
directory to load and store macros
.It Sy --rowkey-prefix Ns = Ns Ar "str " No "\[u30AD]"
string prefix for rowkey in the cmdlog
.It Sy --cmdlog-histfile Ns = Ns Ar "str " No ""
file to autorecord each cmdlog action to
.It Sy --regex-maxsplit Ns = Ns Ar "int " No "0"
maxsplit to pass to regex.split
.It Sy --show-graph-labels Ns = Ns Ar "bool " No "True"
show axes and legend on graph
.It Sy --plot-colors Ns = Ns Ar "str " No ""
list of distinct colors to use for plotting distinct objects
.It Sy --zoom-incr Ns = Ns Ar "float " No "2.0"
amount to multiply current zoomlevel when zooming
.It Sy --motd-url Ns = Ns Ar "str " No ""
source of randomized startup messages
.It Sy --profile Ns = Ns Ar "str " No ""
filename to save binary profiling data
.It Sy --csv-dialect Ns = Ns Ar "str " No "excel"
dialect passed to csv.reader
.It Sy --csv-delimiter Ns = Ns Ar "str " No ","
delimiter passed to csv.reader
.It Sy --csv-quotechar Ns = Ns Ar "str " No """
quotechar passed to csv.reader
.It Sy --csv-skipinitialspace Ns = Ns Ar "bool " No "True"
skipinitialspace passed to csv.reader
.It Sy --csv-escapechar Ns = Ns Ar "NoneType " No "None"
escapechar passed to csv.reader
.It Sy --safety-first No " False"
sanitize input/output to handle edge cases, with a performance cost
.It Sy --json-indent Ns = Ns Ar "NoneType " No "None"
indent to use when saving json
.It Sy --fixed-rows Ns = Ns Ar "int " No "1000"
number of rows to check for fixed width columns
.It Sy --pcap-internet Ns = Ns Ar "str " No "n"
(y/s/n) if save_dot includes all internet hosts separately (y), combined (s), or does not include the internet (n)
.It Sy --graphviz-edge-labels Ns = Ns Ar "bool " No "True"
whether to include edge labels on graphviz diagrams
.El
.
.Ss DISPLAY OPTIONS
.No Display options can only be set via the Sx Options Sheet No or a Pa .visidatarc No (see Sx FILES Ns ).
.Pp
.
.Bl -tag -width XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -compact
.It Sy "disp_note_none " No "\[u2300]"
visible contents of a cell whose value is None
.It Sy "disp_truncator " No "\[u2026]"
indicator that the contents are only partially visible
.It Sy "disp_oddspace " No "\[u00B7]"
displayable character for odd whitespace
.It Sy "disp_unprintable " No "."
substitute character for unprintables
.It Sy "disp_column_sep " No "|"
separator between columns
.It Sy "disp_keycol_sep " No "\[u2016]"
separator between key columns and rest of columns
.It Sy "disp_status_fmt " No "{sheet.name}| "
status line prefix
.It Sy "disp_lstatus_max " No "0"
maximum length of left status line
.It Sy "disp_status_sep " No " | "
separator between statuses
.It Sy "disp_edit_fill " No "_"
edit field fill character
.It Sy "disp_more_left " No "<"
header note indicating more columns to the left
.It Sy "disp_more_right " No ">"
header note indicating more columns to the right
.It Sy "disp_error_val " No ""
displayed contents for computation exception
.It Sy "disp_ambig_width " No "1"
width to use for unicode chars marked ambiguous
.It Sy "color_default " No "normal"
the default color
.It Sy "color_default_hdr " No "bold underline"
color of the column headers
.It Sy "color_current_row " No "reverse"
color of the cursor row
.It Sy "color_current_col " No "bold"
color of the cursor column
.It Sy "color_current_hdr " No "bold reverse underline"
color of the header for the cursor column
.It Sy "color_column_sep " No "246 blue"
color of column separators
.It Sy "color_key_col " No "81 cyan"
color of key columns
.It Sy "color_hidden_col " No "8"
color of hidden columns on metasheets
.It Sy "color_selected_row " No "215 yellow"
color of selected rows
.It Sy "color_keystrokes " No "white"
color of input keystrokes on status line
.It Sy "color_status " No "bold"
status line color
.It Sy "color_error " No "red"
error message color
.It Sy "color_warning " No "yellow"
warning message color
.It Sy "color_edit_cell " No "normal"
cell color to use when editing cell
.It Sy "disp_pending " No ""
string to display in pending cells
.It Sy "color_note_pending " No "bold magenta"
color of note in pending cells
.It Sy "color_note_type " No "226 yellow"
cell note for numeric types in anytype columns
.It Sy "disp_date_fmt " No "%Y-%m-%d"
default fmtstr to strftime for date values
.It Sy "color_change_pending" No "reverse yellow"
color for file attributes pending modification
.It Sy "color_delete_pending" No "red"
color for files pending delete
.It Sy "disp_histogram " No "*"
histogram element character
.It Sy "disp_histolen " No "50"
width of histogram column
.It Sy "color_working " No "green"
color of system running smoothly
.It Sy "disp_replay_play " No "\[u25B6]"
status indicator for active replay
.It Sy "disp_replay_pause " No "\[u2016]"
status indicator for paused replay
.It Sy "color_status_replay" No "green"
color of replay status indicator
.It Sy "disp_pixel_random " No "False"
randomly choose attr from set of pixels instead of most common
.It Sy "color_graph_hidden " No "238 blue"
color of legend for hidden attribute
.It Sy "color_graph_selected" No "bold"
color of selected graph points
.It Sy "color_graph_axis " No "bold"
color for graph axis labels
.It Sy "color_diff " No "red"
color of values different from --diff source
.It Sy "color_diff_add " No "yellow"
color of rows/columns added to --diff source
.El
.
.Sh EXAMPLES
.Dl Nm vd Cm foo.tsv
.No open the file foo.tsv in the current directory
.Pp
.Dl Nm vd Cm -f sqlite bar.db
.No open the file bar.db as a sqlite database
.Pp
.Dl Nm vd Cm -b countries.fixed -o countries.tsv
.No convert countries.fixed (in fixed width format) to countries.tsv (in tsv format)
.Pp
.Dl Nm vd Cm postgres:// Ns Ar username Ns Sy "\&:" Ns Ar password Ns Sy @ Ns Ar hostname Ns Sy "\&:" Ns Ar port Ns Sy / Ns Ar database
.No open a connection to the given postgres database
.Pp
.Dl Nm vd Cm --play tests/pivot.vd --replay-wait 1 --output tests/pivot.tsv
.No replay tests/pivot.vd, waiting 1 second between commands, and output the final sheet to test/pivot.tsv
.Pp
.Dl Ic ls -l | Nm vd Cm -f fixed --skip 1 --header 0
.No parse the output of ls -l into usable data
.Pp
.Sh FILES
At the start of every session,
.Sy VisiData No looks for Pa $HOME/.visidatarc Ns , and calls Python exec() on its contents if it exists.
For example:
.Bd -literal
options.min_memory_mb=100 # stop processing without 100MB free
bindkey('0', 'go-leftmost') # alias '0' to move to first column, like vim
def median(values):
L = sorted(values)
return L[len(L)//2]
aggregator('median', median)
.Ed
.Pp
Functions defined in .visidatarc are available in python expressions (e.g. in derived columns).
.
.Sh SUPPORTED SOURCES
These are the supported sources:
.Pp
.Bl -inset -compact -offset xxx
.It Sy tsv No (tab-separated value)
.Bl -inset -compact -offset xxx
.It Plain and simple. Nm VisiData No writes tsv format by default. See the Sy --delimiter No option.
.El
.El
.Pp
.Bl -inset -compact -offset xxx
.It Sy csv No (comma-separated value)
.Bl -inset -compact -offset xxx
.It .csv files are a scourge upon the earth, and still regrettably common.
.It See the Sy --csv-dialect Ns , Sy --csv-delimiter Ns , Sy --csv-quotechar Ns , and Sy --csv-skipinitialspace No options.
.It Accepted dialects are Ic excel-tab Ns , Ic unix Ns , and Ic excel Ns .
.El
.El
.Pp
.Bl -inset -compact -offset xxx
.It Sy fixed No (fixed width text)
.Bl -inset -compact -offset xxx
.It Columns are autodetected from the first 1000 rows (adjustable with Sy --fixed-rows Ns ).
.El
.El
.Pp
.Bl -inset -compact -offset xxx
.It Sy json No (single object) and Sy jsonl No (one object per line).
.Bl -inset -compact -offset xxx
.It Cells containing lists (e.g. Sy [3] Ns ) or dicts ( Ns Sy {3} Ns ) can be expanded into new columns with Sy "\&(" No and unexpanded with Sy "\&)" Ns .
.El
.El
.Pp
.Bl -inset -compact -offset xxx
.It Sy yaml Ns / Ns Sy yml No (requires Sy PyYAML Ns )
.El
.Pp
.Bl -inset -compact -offset xxx
.It Sy pcap No ( requires Sy xpkt Ns , Sy dnslib Ns )
.Bl -inset -compact -offset xxx
.It View and investigate captured network traffic in a tabular format.
.El
.El
.Pp
.Bl -inset -compact -offset xxx
.It Sy png No (requires Sy pypng Ns )
.Bl -inset -compact -offset xxx
.It Pixels can be edited and saved in data form. Images can be plotted with Ic "\&." No (dot).
.El
.El
.
.Pp
The following URL schemes are supported:
.Bl -inset -compact -offset xxx
.It Sy http No (requires Sy requests Ns ); can be used as transport for with another filetype
.It Sy postgres No (requires Sy psycopg2 Ns )
.El
.
.Pp
.Bl -inset -compact
.It The following sources may include multiple tables. The initial sheet is the table directory;
.Sy Enter No loads the entire table into memory.
.El
.
.Pp
.Bl -inset -compact -offset xxx
.It Sy sqlite
.It Sy xlsx No (requires Sy openpyxl Ns )
.It Sy xls No (requires Sy xlrd Ns )
.It Sy hdf5 No (requires Sy h5py Ns )
.It Sy ttf Ns / Ns Sy otf No (requires Sy fonttools Ns )
.It Sy mbtiles No (requires Sy mapbox-vector-tile Ns )
.It Sy htm Ns / Ns Sy html No (requires Sy lxml Ns )
.It Sy xml No (requires Sy lxml Ns )
.Bl -tag -width XXXX -compact -offset XXX
.It Sy " v"
show only columns in current row attributes
.It Sy za
add column for xml attributes
.El
.It Sy xpt No (SAS; requires Sy xport Ns )
.It Sy sas7bdat No (SAS; requires Sy sas7bdat Ns )
.It Sy sav No (SPSS; requires Sy savReaderWriter Ns )
.It Sy dta No (Stata; requires Sy pandas Ns )
.It Sy shp No (requires Sy pyshp Ns )
.El
.Pp
In addition,
.Sy .zip Ns , Sy .gz Ns , Sy .bz2 Ns , and Sy .xz No files are decompressed on the fly.
.Pp
.No VisiData has an adapter for Sy pandas Ns . To load a file format which is supported by Sy pandas Ns , pass Sy -f pandas data.foo Ns . This will call Sy pandas.read_foo() Ns .
.Pp
.No For example, Sy vd -f pandas data.parquet No loads a parquet file. Note that when using the Sy pandas No loader, the Sy .fileformat No file extension is mandatory
.
.Sh SUPPORTED OUTPUT FORMATS
These are the supported savers:
.Pp
.Bl -inset -compact -offset xxx
.It Sy tsv No (tab-separated value)
.It Sy csv No (comma-separated value)
.It Sy json No (one object with all rows)
.Bl -inset -compact -offset xxx
.It All expanded subcolumns must be closed (with Sy "\&)" Ns ) to retain the same structure.
.It Sy .shp No files can be saved as Sy geoJSON Ns .
.El
.It Sy md No (org-mode compatible markdown table)
.It Sy htm Ns / Ns Sy html No (requires Sy lxml Ns )
.It Sy png No (requires Sy pypng Ns )
.El
.Pp
.No Multisave is supported by Sy html Ns , Sy md Ns , and Sy txt Ns ; Sy g^S No will save all sheets into a single output file.
.Pp
.
.Sh AUTHOR
.Nm VisiData
was made by
.An Saul Pwanson Aq Mt vd@saul.pw Ns .
visidata-1.5.2/visidata/selection.py 0000660 0001750 0001750 00000004247 13416252050 020161 0 ustar kefala kefala 0000000 0000000 from visidata import Sheet
Sheet.addCommand('t', 'stoggle-row', 'toggle([cursorRow]); cursorDown(1)'),
Sheet.addCommand('s', 'select-row', 'selectRow(cursorRow); cursorDown(1)'),
Sheet.addCommand('u', 'unselect-row', 'unselect([cursorRow]); cursorDown(1)'),
Sheet.addCommand('gt', 'stoggle-rows', 'toggle(rows)'),
Sheet.addCommand('gs', 'select-rows', 'select(rows)'),
Sheet.addCommand('gu', 'unselect-rows', '_selectedRows.clear()'),
Sheet.addCommand('zt', 'stoggle-before', 'toggle(rows[:cursorRowIndex])'),
Sheet.addCommand('zs', 'select-before', 'select(rows[:cursorRowIndex])'),
Sheet.addCommand('zu', 'unselect-before', 'unselect(rows[:cursorRowIndex])'),
Sheet.addCommand('gzt', 'stoggle-after', 'toggle(rows[cursorRowIndex:])'),
Sheet.addCommand('gzs', 'select-after', 'select(rows[cursorRowIndex:])'),
Sheet.addCommand('gzu', 'unselect-after', 'unselect(rows[cursorRowIndex:])'),
Sheet.addCommand('|', 'select-col-regex', 'selectByIdx(vd.searchRegex(sheet, regex=input("|", type="regex", defaultLast=True), columns="cursorCol"))'),
Sheet.addCommand('\\', 'unselect-col-regex', 'unselectByIdx(vd.searchRegex(sheet, regex=input("\\\\", type="regex", defaultLast=True), columns="cursorCol"))'),
Sheet.addCommand('g|', 'select-cols-regex', 'selectByIdx(vd.searchRegex(sheet, regex=input("g|", type="regex", defaultLast=True), columns="visibleCols"))'),
Sheet.addCommand('g\\', 'unselect-cols-regex', 'unselectByIdx(vd.searchRegex(sheet, regex=input("g\\\\", type="regex", defaultLast=True), columns="visibleCols"))'),
Sheet.addCommand(',', 'select-equal-cell', 'select(gatherBy(lambda r,c=cursorCol,v=cursorTypedValue: c.getTypedValue(r) == v), progress=False)'),
Sheet.addCommand('g,', 'select-equal-row', 'select(gatherBy(lambda r,currow=cursorRow,vcols=visibleCols: all([c.getTypedValue(r) == c.getTypedValue(currow) for c in vcols])), progress=False)'),
Sheet.addCommand('z|', 'select-expr', 'expr=inputExpr("select by expr: "); select(gatherBy(lambda r, sheet=sheet, expr=expr: sheet.evalexpr(expr, r)), progress=False)'),
Sheet.addCommand('z\\', 'unselect-expr', 'expr=inputExpr("unselect by expr: "); unselect(gatherBy(lambda r, sheet=sheet, expr=expr: sheet.evalexpr(expr, r)), progress=False)')
visidata-1.5.2/visidata/colors.py 0000660 0001750 0001750 00000001504 13416252050 017466 0 ustar kefala kefala 0000000 0000000 #!/usr/bin/env python3
import curses
from visidata import globalCommand, colors, Sheet, Column, RowColorizer, wrapply
globalCommand(None, 'colors', 'vd.push(ColorSheet("vdcolors"))')
class ColorSheet(Sheet):
rowtype = 'colors' # rowdef: color number as assigned in the colors object
columns = [
Column('color', type=int),
Column('R', getter=lambda col,row: curses.color_content(curses.pair_number(colors[row])-1)[0]),
Column('G', getter=lambda col,row: curses.color_content(curses.pair_number(colors[row])-1)[1]),
Column('B', getter=lambda col,row: curses.color_content(curses.pair_number(colors[row])-1)[2]),
]
colorizers = [
RowColorizer(7, None, lambda s,c,r,v: r)
]
def reload(self):
self.rows = sorted(colors.keys(), key=lambda n: wrapply(int, n))
visidata-1.5.2/visidata/slide.py 0000660 0001750 0001750 00000003717 13416252050 017275 0 ustar kefala kefala 0000000 0000000 '''slide rows/columns around'''
from visidata import Sheet, moveListItem, globalCommand
Sheet.addCommand('H', 'slide-left', 'i = sheet.cursorVisibleColIndex; sheet.cursorVisibleColIndex = moveVisibleCol(sheet, i, i-1)')
Sheet.addCommand('L', 'slide-right', 'i = sheet.cursorVisibleColIndex; sheet.cursorVisibleColIndex = moveVisibleCol(sheet, i, i+1)')
Sheet.addCommand('J', 'slide-down', 'i = sheet.cursorRowIndex; sheet.cursorRowIndex = moveListItem(rows, i, i+1)')
Sheet.addCommand('K', 'slide-up', 'i = sheet.cursorRowIndex; sheet.cursorRowIndex = moveListItem(rows, i, i-1)')
Sheet.addCommand('gH', 'slide-leftmost', 'columns.insert(0, columns.pop(cursorColIndex))')
Sheet.addCommand('gL', 'slide-rightmost', 'columns.append(columns.pop(cursorColIndex))')
Sheet.addCommand('gJ', 'slide-bottom', 'rows.append(rows.pop(cursorRowIndex))')
Sheet.addCommand('gK', 'slide-top', 'rows.insert(0, rows.pop(cursorRowIndex))')
Sheet.addCommand('zH', 'slide-left-n', 'i = sheet.cursorVisibleColIndex; n=int(input("slide col left n=", value=1)); sheet.cursorVisibleColIndex = moveVisibleCol(sheet, i, i-n)')
Sheet.addCommand('zL', 'slide-right-n', 'i = sheet.cursorVisibleColIndex; n=int(input("slide col right n=", value=1)); sheet.cursorVisibleColIndex = moveVisibleCol(sheet, i, i+n)')
Sheet.addCommand('zJ', 'slide-down-n', 'i = sheet.cursorRowIndex; n=int(input("slide row down n=", value=1)); sheet.cursorRowIndex = moveListItem(rows, i, i+n)')
Sheet.addCommand('zK', 'slide-up-n', 'i = sheet.cursorRowIndex; n=int(input("slide row up n=", value=1)); sheet.cursorRowIndex = moveListItem(rows, i, i-n)')
def moveVisibleCol(sheet, fromVisColIdx, toVisColIdx):
'Move visible column to another visible index in sheet.'
toVisColIdx = min(max(toVisColIdx, 0), sheet.nVisibleCols)
fromColIdx = sheet.columns.index(sheet.visibleCols[fromVisColIdx])
toColIdx = sheet.columns.index(sheet.visibleCols[toVisColIdx])
moveListItem(sheet.columns, fromColIdx, toColIdx)
return toVisColIdx
visidata-1.5.2/visidata/dev.py 0000660 0001750 0001750 00000000775 13416252050 016754 0 ustar kefala kefala 0000000 0000000 from visidata import *
class StatusMaker:
def __init__(self, name, *args, **kwargs):
self._name = name
def __getattr__(self, k):
return StatusMaker(status('%s.%s' % (self._name, k)))
# def __setattr__(self, k, v):
# super().__setattr__(k, v)
# if k != '_name':
# status('%s.%s := %s' % (self._name, k, v))
def __call__(self, *args, **kwargs):
return StatusMaker(status('%s(%s, %s)' % (self._name, ', '.join(str(x) for x in args), kwargs)))
visidata-1.5.2/visidata/errors.py 0000660 0001750 0001750 00000000756 13416252050 017511 0 ustar kefala kefala 0000000 0000000 from visidata import globalCommand, Sheet, TextSheet, vd, error, stacktrace
globalCommand('^E', 'error-recent', 'vd.lastErrors and vd.push(ErrorSheet("last_error", vd.lastErrors[-1])) or status("no error")')
globalCommand('g^E', 'errors-all', 'vd.push(ErrorSheet("last_errors", sum(vd.lastErrors[-10:], [])))')
Sheet.addCommand('z^E', 'error-cell', 'vd.push(ErrorSheet("cell_error", getattr(cursorCell, "error", None) or fail("no error this cell")))')
class ErrorSheet(TextSheet):
pass
visidata-1.5.2/visidata/_profile.py 0000660 0001750 0001750 00000010404 13416500243 017763 0 ustar kefala kefala 0000000 0000000 import os.path
import functools
import cProfile
import threading
import collections
from visidata import vd, option, options, status, globalCommand, Sheet, EscapeException
from visidata import elapsed_s, ColumnAttr, Column, ThreadsSheet, ENTER
option('profile', '', 'filename to save binary profiling data')
globalCommand('^_', 'toggle-profile', 'toggleProfiling(threading.current_thread())')
ThreadsSheet.addCommand(ENTER, 'profile-row', 'vd.push(ProfileSheet(cursorRow.name+"_profile", source=cursorRow.profile.getstats()))')
min_thread_time_s = 0.10 # only keep threads that take longer than this number of seconds
def open_pyprof(p):
return ProfileSheet(p.name, p.open_bytes())
def toggleProfiling(t):
if not t.profile:
t.profile = cProfile.Profile()
t.profile.enable()
if not options.profile:
options.set('profile', 'vdprofile')
else:
t.profile.disable()
t.profile = None
options.set('profile', '')
status('profiling ' + ('ON' if t.profile else 'OFF'))
@functools.wraps(vd().toplevelTryFunc)
def threadProfileCode(func, *args, **kwargs):
'Toplevel thread profile wrapper.'
with ThreadProfiler(threading.current_thread()) as prof:
try:
prof.thread.status = threadProfileCode.__wrapped__(func, *args, **kwargs)
except EscapeException as e:
prof.thread.status = e
class ThreadProfiler:
numProfiles = 0
def __init__(self, thread):
self.thread = thread
if options.profile:
self.thread.profile = cProfile.Profile()
else:
self.thread.profile = None
ThreadProfiler.numProfiles += 1
self.profileNumber = ThreadProfiler.numProfiles
def __enter__(self):
if self.thread.profile:
self.thread.profile.enable()
return self
def __exit__(self, exc_type, exc_val, tb):
if self.thread.profile:
self.thread.profile.disable()
self.thread.profile.dump_stats(options.profile + str(self.profileNumber))
if exc_val:
self.thread.exception = exc_val
else:
# remove very-short-lived async actions
if elapsed_s(self.thread) < min_thread_time_s:
vd().threads.remove(self.thread)
class ProfileSheet(Sheet):
columns = [
Column('funcname', getter=lambda col,row: codestr(row.code)),
Column('filename', getter=lambda col,row: os.path.split(row.code.co_filename)[-1] if not isinstance(row.code, str) else ''),
Column('linenum', type=int, getter=lambda col,row: row.code.co_firstlineno if not isinstance(row.code, str) else None),
Column('inlinetime_us', type=int, getter=lambda col,row: row.inlinetime*1000000),
Column('totaltime_us', type=int, getter=lambda col,row: row.totaltime*1000000),
ColumnAttr('callcount', type=int),
Column('avg_inline_us', type=int, getter=lambda col,row: row.inlinetime*1000000/row.callcount),
Column('avg_total_us', type=int, getter=lambda col,row: row.totaltime*1000000/row.callcount),
ColumnAttr('reccallcount', type=int),
ColumnAttr('calls'),
Column('callers', getter=lambda col,row: col.sheet.callers[row.code]),
]
nKeys=3
def reload(self):
self.rows = self.source
self.orderBy(self.column('inlinetime_us'), reverse=True)
self.callers = collections.defaultdict(list) # [row.code] -> list(code)
for r in self.rows:
calls = getattr(r, 'calls', None)
if calls:
for callee in calls:
self.callers[callee.code].append(r)
def codestr(code):
if isinstance(code, str):
return code
return code.co_name
ProfileSheet.addCommand('z^S', 'save-profile', 'profile.dump_stats(input("save profile to: ", value=name+".prof"))')
ProfileSheet.addCommand(ENTER, 'dive-row', 'vd.push(ProfileSheet(codestr(cursorRow.code)+"_calls", source=cursorRow.calls or fail("no calls")))')
ProfileSheet.addCommand('z'+ENTER, 'dive-cell', 'vd.push(ProfileSheet(codestr(cursorRow.code)+"_"+cursorCol.name, source=cursorValue or fail("no callers")))')
ProfileSheet.addCommand('^O', 'sysopen-row', 'launchEditor(cursorRow.code.co_filename, "+%s" % cursorRow.code.co_firstlineno)')
vd.toplevelTryFunc = threadProfileCode
visidata-1.5.2/visidata/cmdlog.py 0000660 0001750 0001750 00000032310 13416500243 017431 0 ustar kefala kefala 0000000 0000000 import threading
from visidata import *
import visidata
option('replay_wait', 0.0, 'time to wait between replayed commands, in seconds')
theme('disp_replay_play', 'â–¶', 'status indicator for active replay')
theme('disp_replay_pause', '‖', 'status indicator for paused replay')
theme('color_status_replay', 'green', 'color of replay status indicator')
option('replay_movement', False, 'insert movements during replay')
option('visidata_dir', '~/.visidata/', 'directory to load and store macros')
globalCommand('gD', 'visidata-dir', 'p=Path(options.visidata_dir); vd.push(DirSheet(str(p), source=p))')
globalCommand('D', 'cmdlog', 'vd.push(vd.cmdlog)')
globalCommand('^D', 'save-cmdlog', 'saveSheets(inputFilename("save to: ", value=fnSuffix("cmdlog-{0}.vd") or "cmdlog.vd"), vd.cmdlog)')
globalCommand('^U', 'pause-replay', 'CommandLog.togglePause()')
globalCommand('^I', 'advance-replay', '(CommandLog.currentReplay or fail("no replay to advance")).advance()')
globalCommand('^K', 'stop-replay', '(CommandLog.currentReplay or fail("no replay to cancel")).cancel()')
globalCommand('Q', 'forget-sheet', 'vd.cmdlog.removeSheet(vd.sheets.pop(0))')
globalCommand(None, 'status', 'status(input("status: "))')
globalCommand('^V', 'check-version', 'status(__version_info__); checkVersion(input("require version: ", value=__version_info__))')
# not necessary to log movements and scrollers
nonLogKeys = 'KEY_DOWN KEY_UP KEY_NPAGE KEY_PPAGE j k gj gk ^F ^B r < > { } / ? n N gg G g/ g? g_ _ z_'.split()
nonLogKeys += 'KEY_LEFT KEY_RIGHT h l gh gl c Q'.split()
nonLogKeys += 'zk zj zt zz zb zh zl zKEY_LEFT zKEY_RIGHT'.split()
nonLogKeys += '^^ ^Z ^A ^L ^C ^U ^K ^I ^D ^G KEY_RESIZE KEY_F(1) ^H KEY_BACKSPACE'.split()
nonLogKeys += [' ']
option('rowkey_prefix', 'ã‚', 'string prefix for rowkey in the cmdlog')
option('cmdlog_histfile', '', 'file to autorecord each cmdlog action to')
def checkVersion(desired_version):
if desired_version != visidata.__version__:
fail("version %s required" % desired_version)
def fnSuffix(template):
for i in range(1, 1000):
fn = template.format(i)
if not Path(fn).exists():
return fn
def indexMatch(L, func):
'returns the smallest i for which func(L[i]) is true'
for i, x in enumerate(L):
if func(x):
return i
def keystr(k):
return options.rowkey_prefix + ','.join(map(str, k))
def isLoggableSheet(sheet):
return sheet is not vd().cmdlog and not isinstance(sheet, (OptionsSheet, ErrorSheet))
def isLoggableCommand(keystrokes, longname):
if keystrokes in nonLogKeys:
return False
if longname.startswith('go-'):
return False
if keystrokes.startswith('BUTTON'): # mouse click
return False
if keystrokes.startswith('REPORT'): # scrollwheel/mouse position
return False
return True
def open_vd(p):
return CommandLog(p.name, source=p)
save_vd = save_tsv
# rowdef: namedlist (like TsvSheet)
class CommandLog(TsvSheet):
'Log of commands for current session.'
rowtype = 'logged commands'
precious = False
_rowtype = namedlist('CommandLogRow', 'sheet col row longname input keystrokes comment'.split())
columns = [ColumnAttr(x) for x in _rowtype._fields]
paused = False
currentReplay = None # CommandLog replaying currently
currentReplayRow = None # must be global, to allow replay
semaphore = threading.Semaphore(0)
filetype = 'vd'
def __init__(self, name, source=None, **kwargs):
super().__init__(name, source=source, **kwargs)
options.set('delimiter', '\t', self)
self.currentActiveRow = None
def newRow(self, **fields):
return self._rowtype(**fields)
def removeSheet(self, vs):
'Remove all traces of sheets named vs.name from the cmdlog.'
self.rows = [r for r in self.rows if r.sheet != vs.name]
status('removed "%s" from cmdlog' % vs.name)
def saveMacro(self, rows, ks):
vs = copy(self)
vs.rows = self.selectedRows
macropath = Path(fnSuffix(options.visidata_dir+"macro-{0}.vd"))
save_vd(macropath, vs)
setMacro(ks, vs)
append_tsv_row(vd().macrosheet, (ks, macropath.resolve()))
def beforeExecHook(self, sheet, cmd, args, keystrokes):
if not isLoggableSheet(sheet):
return # don't record editlog commands
if self.currentActiveRow:
self.afterExecSheet(sheet, False, '')
sheetname, colname, rowname = '', '', ''
if sheet and cmd.longname != 'open-file':
contains = lambda s, *substrs: any((a in s) for a in substrs)
sheetname = sheet.name
if contains(cmd.execstr, 'cursorTypedValue', 'cursorDisplay', 'cursorValue', 'cursorCell', 'cursorRow') and sheet.rows:
k = sheet.rowkey(sheet.cursorRow)
rowname = keystr(k) if k else sheet.cursorRowIndex
if contains(cmd.execstr, 'cursorTypedValue', 'cursorDisplay', 'cursorValue', 'cursorCell', 'cursorCol', 'cursorVisibleCol'):
colname = sheet.cursorCol.name or sheet.visibleCols.index(sheet.cursorCol)
comment = CommandLog.currentReplayRow.comment if CommandLog.currentReplayRow else cmd.helpstr
self.currentActiveRow = self.newRow(sheet=sheetname, col=colname, row=rowname,
keystrokes=keystrokes, input=args,
longname=cmd.longname, comment=comment)
def afterExecSheet(self, sheet, escaped, err):
'Records currentActiveRow'
if not self.currentActiveRow: # nothing to record
return
if err:
self.currentActiveRow[-1] += ' [%s]' % err
if isLoggableSheet(sheet): # don't record jumps to cmdlog or other internal sheets
# remove user-aborted commands and simple movements
if not escaped and isLoggableCommand(self.currentActiveRow.keystrokes, self.currentActiveRow.longname):
self.addRow(self.currentActiveRow)
if options.cmdlog_histfile:
if not getattr(vd(), 'sessionlog', None):
vd().sessionlog = loadInternalSheet(CommandLog, Path(date().strftime(options.cmdlog_histfile)))
append_tsv_row(vd().sessionlog, self.currentActiveRow)
self.currentActiveRow = None
def openHook(self, vs, src):
self.addRow(self.newRow(keystrokes='o', input=src, longname='open-file'))
@classmethod
def togglePause(self):
if not CommandLog.currentReplay:
status('no replay to pause')
else:
if self.paused:
CommandLog.currentReplay.advance()
self.paused = not self.paused
status('paused' if self.paused else 'resumed')
def advance(self):
CommandLog.semaphore.release()
def cancel(self):
CommandLog.currentReplayRow = None
CommandLog.currentReplay = None
self.advance()
def moveToReplayContext(self, r):
'set the sheet/row/col to the values in the replay row. return sheet'
if not r.sheet:
# assert not r.col and not r.row
return self # any old sheet should do, row/column don't matter
try:
sheetidx = int(r.sheet)
vs = vd().sheets[sheetidx]
except ValueError:
vs = vd().getSheet(r.sheet) or error('no sheet named %s' % r.sheet)
if r.row:
try:
rowidx = int(r.row)
except ValueError:
rowidx = indexMatch(vs.rows, lambda r,vs=vs,k=r.row: keystr(vs.rowkey(r)) == k)
if rowidx is None:
error('no "%s" row' % r.row)
if options.replay_movement:
while vs.cursorRowIndex != rowidx:
vs.cursorRowIndex += 1 if (rowidx - vs.cursorRowIndex) > 0 else -1
while not self.delay(0.5):
pass
else:
vs.cursorRowIndex = rowidx
if r.col:
try:
vcolidx = int(r.col)
except ValueError:
vcolidx = indexMatch(vs.visibleCols, lambda c,name=r.col: name == c.name)
if vcolidx is None:
error('no "%s" column' % r.col)
if options.replay_movement:
while vs.cursorVisibleColIndex != vcolidx:
vs.cursorVisibleColIndex += 1 if (vcolidx - vs.cursorVisibleColIndex) > 0 else -1
while not self.delay(0.5):
pass
assert vs.cursorVisibleColIndex == vcolidx
else:
vs.cursorVisibleColIndex = vcolidx
return vs
def delay(self, factor=1):
'returns True if delay satisfied'
acquired = CommandLog.semaphore.acquire(timeout=options.replay_wait*factor if not self.paused else None)
return acquired or not self.paused
def replayOne(self, r):
'Replay the command in one given row.'
CommandLog.currentReplayRow = r
longname = getattr(r, 'longname', None)
if longname == 'set-option':
try:
options.set(r.row, r.input, options._opts.getobj(r.col))
escaped = False
except Exception as e:
exceptionCaught(e)
escaped = True
else:
vs = self.moveToReplayContext(r)
vd().keystrokes = r.keystrokes
# <=v1.2 used keystrokes in longname column; getCommand fetches both
escaped = vs.exec_command(vs.getCommand(longname if longname else r.keystrokes), keystrokes=r.keystrokes)
CommandLog.currentReplayRow = None
if escaped: # escape during replay aborts replay
warning('replay aborted')
return escaped
def replay_sync(self, live=False):
'Replay all commands in log.'
self.cursorRowIndex = 0
CommandLog.currentReplay = self
with Progress(total=len(self.rows)) as prog:
while self.cursorRowIndex < len(self.rows):
if CommandLog.currentReplay is None:
status('replay canceled')
return
vd().statuses.clear()
try:
if self.replayOne(self.cursorRow):
self.cancel()
return
except Exception as e:
self.cancel()
exceptionCaught(e)
status('replay canceled')
return
self.cursorRowIndex += 1
prog.addProgress(1)
sync(1 if live else 0) # expect this thread also if playing live
while not self.delay():
pass
status('replay complete')
CommandLog.currentReplay = None
@asyncthread
def replay(self):
'Inject commands into live execution with interface.'
self.replay_sync(live=True)
def getLastArgs(self):
'Get user input for the currently playing command.'
if CommandLog.currentReplayRow:
return CommandLog.currentReplayRow.input
return None
def setLastArgs(self, args):
'Set user input on last command, if not already set.'
# only set if not already set (second input usually confirmation)
if self.currentActiveRow is not None:
if not self.currentActiveRow.input:
self.currentActiveRow.input = args
@property
def replayStatus(self):
x = options.disp_replay_pause if self.paused else options.disp_replay_play
return ' │ %s %s/%s' % (x, self.cursorRowIndex, len(self.rows))
def setOption(self, optname, optval, obj=None):
objname = options._opts.objname(obj)
self.addRow(self.newRow(col=objname, row=optname,
keystrokes='', input=str(optval),
longname='set-option'))
CommandLog.addCommand('x', 'replay-row', 'sheet.replayOne(cursorRow); status("replayed one row")')
CommandLog.addCommand('gx', 'replay-all', 'sheet.replay()')
CommandLog.addCommand('^C', 'stop-replay', 'sheet.cursorRowIndex = sheet.nRows')
CommandLog.addCommand('z^S', 'save-macro', 'sheet.saveMacro(selectedRows or fail("no rows selected"), input("save macro for keystroke: "))')
options.set('header', 1, CommandLog) # .vd files always have a header row, regardless of options
vd().cmdlog = CommandLog('cmdlog')
vd().cmdlog.rows = []
vd().addHook('preexec', vd().cmdlog.beforeExecHook)
vd().addHook('postexec', vd().cmdlog.afterExecSheet)
vd().addHook('preedit', vd().cmdlog.getLastArgs)
vd().addHook('postedit', vd().cmdlog.setLastArgs)
vd().addHook('rstatus', lambda sheet: CommandLog.currentReplay and (CommandLog.currentReplay.replayStatus, 'color_status_replay'))
vd().addHook('set_option', vd().cmdlog.setOption)
def loadMacros():
macrospath = Path(os.path.join(options.visidata_dir, 'macros.tsv'))
macrosheet = loadInternalSheet(TsvSheet, macrospath, columns=(ColumnItem('command', 0), ColumnItem('filename', 1))) or error('error loading macros')
for ks, fn in macrosheet.rows:
vs = loadInternalSheet(CommandLog, Path(fn))
setMacro(ks, vs)
return macrosheet
def setMacro(ks, vs):
bindkeys.set(ks, vs.name, 'override')
commands.set(vs.name, vs, 'override')
vd().macrosheet = loadMacros()
visidata-1.5.2/visidata/utils.py 0000660 0001750 0001750 00000001353 13416500243 017327 0 ustar kefala kefala 0000000 0000000
def joinSheetnames(*sheetnames):
'Concatenate sheet names in a standard way'
return '_'.join(str(x) for x in sheetnames)
def moveListItem(L, fromidx, toidx):
"Move element within list `L` and return element's new index."
r = L.pop(fromidx)
L.insert(toidx, r)
return toidx
class OnExit:
'"with OnExit(func, ...):" calls func(...) when the context is exited'
def __init__(self, func, *args, **kwargs):
self.func = func
self.args = args
self.kwargs = kwargs
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, exc_traceback):
try:
self.func(*self.args, **self.kwargs)
except Exception as e:
exceptionCaught(e)
visidata-1.5.2/visidata/vimkeys.py 0000660 0001750 0001750 00000000537 13416252050 017661 0 ustar kefala kefala 0000000 0000000 from visidata import bindkey
bindkey('h', 'go-left'),
bindkey('j', 'go-down'),
bindkey('k', 'go-up'),
bindkey('l', 'go-right'),
bindkey('^F', 'next-page'),
bindkey('^B', 'prev-page'),
bindkey('gg', 'go-top'),
bindkey('G', 'go-bottom'),
bindkey('gj', 'go-bottom'),
bindkey('gk', 'go-top'),
bindkey('gh', 'go-leftmost'),
bindkey('gl', 'go-rightmost')
visidata-1.5.2/visidata/transpose.py 0000660 0001750 0001750 00000001404 13416252050 020202 0 ustar kefala kefala 0000000 0000000 from visidata import *
Sheet.addCommand('T', 'transpose', 'vd.push(TransposeSheet(name+"_T", source=sheet))')
# rowdef: Column
class TransposeSheet(Sheet):
@asyncthread
def reload(self):
# key rows become column names
self.columns = [
Column('_'.join(c.name for c in self.source.keyCols),
getter=lambda c,origcol: origcol.name)
]
self.setKeys(self.columns)
# rows become columns
for row in Progress(self.source.rows, 'transposing'):
self.addColumn(Column('_'.join(self.source.rowkey(row)),
getter=lambda c,origcol,row=row: origcol.getValue(row)))
# columns become rows
self.rows = list(self.source.nonKeyVisibleCols)
visidata-1.5.2/README.md 0000660 0001750 0001750 00000010734 13416502741 015300 0 ustar kefala kefala 0000000 0000000 # VisiData v1.5.2 [](https://circleci.com/gh/saulpw/visidata/tree/stable)
A terminal interface for exploring and arranging tabular data.
## Dependencies
- Linux or OS/X
- Python 3.4+
- python-dateutil
- other modules may be required for opening particular data sources
- see [requirements.txt](https://github.com/saulpw/visidata/blob/stable/requirements.txt) or the [supported sources](https://visidata.org/man/#loaders) in the vd manpage
## Getting started
### Installation
Each package contains the full loader suite but differs in which loader dependencies will get installed by default.
The base VisiData package concerns loaders whose dependencies are covered by the Python3 standard library.
Base loaders: tsv, csv, json, sqlite, and fixed width text.
|Platform |Package Manager | Command | Out-of-box Loaders |
|-------------------|----------------------------------------|----------------------------------------------|----------------------|
|Python3.4+ |[pip3](https://visidata.org/install#pip3) | `pip3 install visidata` | Base |
|Python3.4+ |[conda](https://visidata.org/install#conda) | `conda install --channel conda-forge visidata` | Base, http, html, xls(x) |
|MacOS |[Homebrew](https://visidata.org/install#brew) | `brew install saulpw/vd/visidata` | Base, http, html, xls(x) |
|Linux (Debian/Ubuntu) |[apt](https://visidata.org/install#apt) | [full instructions](https://visidata.org/install#apt) | Base, http, html, xls(x) |
|Linux (Debian/Ubuntu) |[dpkg](https://visidata.org/install#dpkg) | [full instructions](https://visidata.org/install#dpkg) | Base, http, html, xls(x) |
|Windows |[WSL](https://visidata.org/install#wsl) | Windows is not directly supported (use WSL) | N/A |
|Python3.4+ |[github](https://visidata.org/install#git) | `pip3 install git+https://github.com/saulpw/visidata.git@stable` | Base |
Please see [/install](https://visidata.org/install) for detailed instructions, additional information, and troubleshooting.
### Usage
$ vd [] ...
$ | vd []
VisiData supports tsv, csv, xlsx, hdf5, sqlite, json and more (see the [list of supported sources](https://visidata.org/man#sources)).
Use `-f ` to force a particular filetype.
### Documentation
* [Intro to VisiData Tutorial](https://jsvine.github.io/intro-to-visidata/) by [Jeremy Singer-Vine](https://www.jsvine.com/)
* Quick reference: `Ctrl+H` within `vd` will open the [man page](https://visidata.org/man), which has a list of all commands and options.
* [keyboard list of commands](https://visidata.org/docs/kblayout)
* [/docs](https://visidata.org/docs) contains a collection of howto recipes.
### Help and Support
If you have a question, issue, or suggestion regarding VisiData, please [create an issue on Github](https://github.com/saulpw/visidata/issues) or chat with us at #visidata on [freenode.net](https://webchat.freenode.net/).
## vdtui
The core `vdtui.py` can be used to quickly create efficient terminal workflows. These have been prototyped as proof of this concept:
- [vgit](https://github.com/saulpw/vgit): a git interface
- [vsh](https://github.com/saulpw/vsh): a collection of utilities like `vping` and `vtop`.
- [vdgalcon](https://github.com/saulpw/vdgalcon): a port of the classic game [Galactic Conquest](https://www.galcon.com)
Other workflows should also be created as separate apps using vdtui. These apps can be very small and provide a lot of functionality; for example, see the included [viewtsv](bin/viewtsv).
## License
The innermost core file, `vdtui.py`, is a single-file stand-alone library that provides a solid framework for building text user interface apps. It is distributed under the MIT free software license, and freely available for inclusion in other projects.
Other VisiData components, including the main `vd` application, addons, loaders, and other code in this repository, are available for use and distribution under GPLv3.
## Credits
VisiData is conceived and developed by Saul Pwanson ``.
Anja Kefala `` maintains the documentation and packages for all platforms.
Many thanks to numerous other contributors, and to those wonderful users who provide feedback, for helping to make VisiData the awesome tool that it is.
visidata-1.5.2/MANIFEST.in 0000660 0001750 0001750 00000000137 13416252050 015546 0 ustar kefala kefala 0000000 0000000 include README.md
include LICENSE.gpl3
include visidata/man/vd.1
include visidata/commands.tsv
visidata-1.5.2/setup.cfg 0000660 0001750 0001750 00000000046 13416503554 015640 0 ustar kefala kefala 0000000 0000000 [egg_info]
tag_build =
tag_date = 0
visidata-1.5.2/PKG-INFO 0000660 0001750 0001750 00000014307 13416503554 015121 0 ustar kefala kefala 0000000 0000000 Metadata-Version: 1.2
Name: visidata
Version: 1.5.2
Summary: curses interface for exploring and arranging tabular data
Home-page: https://visidata.org
Author: Saul Pwanson
Author-email: visidata@saul.pw
License: GPLv3
Download-URL: https://github.com/saulpw/visidata/tarball/1.5.2
Description: # VisiData v1.5.2 [](https://circleci.com/gh/saulpw/visidata/tree/stable)
A terminal interface for exploring and arranging tabular data.
## Dependencies
- Linux or OS/X
- Python 3.4+
- python-dateutil
- other modules may be required for opening particular data sources
- see [requirements.txt](https://github.com/saulpw/visidata/blob/stable/requirements.txt) or the [supported sources](https://visidata.org/man/#loaders) in the vd manpage
## Getting started
### Installation
Each package contains the full loader suite but differs in which loader dependencies will get installed by default.
The base VisiData package concerns loaders whose dependencies are covered by the Python3 standard library.
Base loaders: tsv, csv, json, sqlite, and fixed width text.
|Platform |Package Manager | Command | Out-of-box Loaders |
|-------------------|----------------------------------------|----------------------------------------------|----------------------|
|Python3.4+ |[pip3](https://visidata.org/install#pip3) | `pip3 install visidata` | Base |
|Python3.4+ |[conda](https://visidata.org/install#conda) | `conda install --channel conda-forge visidata` | Base, http, html, xls(x) |
|MacOS |[Homebrew](https://visidata.org/install#brew) | `brew install saulpw/vd/visidata` | Base, http, html, xls(x) |
|Linux (Debian/Ubuntu) |[apt](https://visidata.org/install#apt) | [full instructions](https://visidata.org/install#apt) | Base, http, html, xls(x) |
|Linux (Debian/Ubuntu) |[dpkg](https://visidata.org/install#dpkg) | [full instructions](https://visidata.org/install#dpkg) | Base, http, html, xls(x) |
|Windows |[WSL](https://visidata.org/install#wsl) | Windows is not directly supported (use WSL) | N/A |
|Python3.4+ |[github](https://visidata.org/install#git) | `pip3 install git+https://github.com/saulpw/visidata.git@stable` | Base |
Please see [/install](https://visidata.org/install) for detailed instructions, additional information, and troubleshooting.
### Usage
$ vd [] ...
$ | vd []
VisiData supports tsv, csv, xlsx, hdf5, sqlite, json and more (see the [list of supported sources](https://visidata.org/man#sources)).
Use `-f ` to force a particular filetype.
### Documentation
* [Intro to VisiData Tutorial](https://jsvine.github.io/intro-to-visidata/) by [Jeremy Singer-Vine](https://www.jsvine.com/)
* Quick reference: `Ctrl+H` within `vd` will open the [man page](https://visidata.org/man), which has a list of all commands and options.
* [keyboard list of commands](https://visidata.org/docs/kblayout)
* [/docs](https://visidata.org/docs) contains a collection of howto recipes.
### Help and Support
If you have a question, issue, or suggestion regarding VisiData, please [create an issue on Github](https://github.com/saulpw/visidata/issues) or chat with us at #visidata on [freenode.net](https://webchat.freenode.net/).
## vdtui
The core `vdtui.py` can be used to quickly create efficient terminal workflows. These have been prototyped as proof of this concept:
- [vgit](https://github.com/saulpw/vgit): a git interface
- [vsh](https://github.com/saulpw/vsh): a collection of utilities like `vping` and `vtop`.
- [vdgalcon](https://github.com/saulpw/vdgalcon): a port of the classic game [Galactic Conquest](https://www.galcon.com)
Other workflows should also be created as separate apps using vdtui. These apps can be very small and provide a lot of functionality; for example, see the included [viewtsv](bin/viewtsv).
## License
The innermost core file, `vdtui.py`, is a single-file stand-alone library that provides a solid framework for building text user interface apps. It is distributed under the MIT free software license, and freely available for inclusion in other projects.
Other VisiData components, including the main `vd` application, addons, loaders, and other code in this repository, are available for use and distribution under GPLv3.
## Credits
VisiData is conceived and developed by Saul Pwanson ``.
Anja Kefala `` maintains the documentation and packages for all platforms.
Many thanks to numerous other contributors, and to those wonderful users who provide feedback, for helping to make VisiData the awesome tool that it is.
Keywords: console tabular data spreadsheet terminal viewer textpunkcurses csv hdf5 h5 xlsx excel tsv
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable
Classifier: Environment :: Console
Classifier: Environment :: Console :: Curses
Classifier: Intended Audience :: Developers
Classifier: Intended Audience :: Science/Research
Classifier: Intended Audience :: System Administrators
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python :: 3
Classifier: Topic :: Database :: Front-Ends
Classifier: Topic :: Scientific/Engineering
Classifier: Topic :: Office/Business :: Financial :: Spreadsheet
Classifier: Topic :: Scientific/Engineering :: Visualization
Classifier: Topic :: Utilities
Requires-Python: >=3.4
visidata-1.5.2/visidata.egg-info/ 0000770 0001750 0001750 00000000000 13416503554 017314 5 ustar kefala kefala 0000000 0000000 visidata-1.5.2/visidata.egg-info/SOURCES.txt 0000777 0001750 0001750 00000003073 13416503554 021215 0 ustar kefala kefala 0000000 0000000 LICENSE.gpl3
MANIFEST.in
README.md
setup.py
bin/vd
visidata/__init__.py
visidata/_profile.py
visidata/_types.py
visidata/aggregators.py
visidata/asyncthread.py
visidata/canvas.py
visidata/clipboard.py
visidata/cmdlog.py
visidata/colors.py
visidata/commands.tsv
visidata/data.py
visidata/describe.py
visidata/dev.py
visidata/diff.py
visidata/errors.py
visidata/freeze.py
visidata/freqtbl.py
visidata/graph.py
visidata/join.py
visidata/metasheets.py
visidata/motd.py
visidata/movement.py
visidata/namedlist.py
visidata/path.py
visidata/pivot.py
visidata/pyobj.py
visidata/regex.py
visidata/search.py
visidata/selection.py
visidata/shell.py
visidata/slide.py
visidata/tidydata.py
visidata/transpose.py
visidata/urlcache.py
visidata/utils.py
visidata/vdtui.py
visidata/vimkeys.py
visidata/zscroll.py
visidata.egg-info/PKG-INFO
visidata.egg-info/SOURCES.txt
visidata.egg-info/dependency_links.txt
visidata.egg-info/requires.txt
visidata.egg-info/top_level.txt
visidata/loaders/__init__.py
visidata/loaders/_pandas.py
visidata/loaders/csv.py
visidata/loaders/fixed_width.py
visidata/loaders/graphviz.py
visidata/loaders/hdf5.py
visidata/loaders/html.py
visidata/loaders/http.py
visidata/loaders/json.py
visidata/loaders/markdown.py
visidata/loaders/mbtiles.py
visidata/loaders/pcap.py
visidata/loaders/png.py
visidata/loaders/postgres.py
visidata/loaders/sas.py
visidata/loaders/shp.py
visidata/loaders/spss.py
visidata/loaders/sqlite.py
visidata/loaders/tsv.py
visidata/loaders/ttf.py
visidata/loaders/xlsx.py
visidata/loaders/xml.py
visidata/loaders/yaml.py
visidata/loaders/zip.py
visidata/man/vd.1 visidata-1.5.2/visidata.egg-info/dependency_links.txt 0000777 0001750 0001750 00000000001 13416503554 023374 0 ustar kefala kefala 0000000 0000000
visidata-1.5.2/visidata.egg-info/PKG-INFO 0000777 0001750 0001750 00000014307 13416503554 020430 0 ustar kefala kefala 0000000 0000000 Metadata-Version: 1.2
Name: visidata
Version: 1.5.2
Summary: curses interface for exploring and arranging tabular data
Home-page: https://visidata.org
Author: Saul Pwanson
Author-email: visidata@saul.pw
License: GPLv3
Download-URL: https://github.com/saulpw/visidata/tarball/1.5.2
Description: # VisiData v1.5.2 [](https://circleci.com/gh/saulpw/visidata/tree/stable)
A terminal interface for exploring and arranging tabular data.
## Dependencies
- Linux or OS/X
- Python 3.4+
- python-dateutil
- other modules may be required for opening particular data sources
- see [requirements.txt](https://github.com/saulpw/visidata/blob/stable/requirements.txt) or the [supported sources](https://visidata.org/man/#loaders) in the vd manpage
## Getting started
### Installation
Each package contains the full loader suite but differs in which loader dependencies will get installed by default.
The base VisiData package concerns loaders whose dependencies are covered by the Python3 standard library.
Base loaders: tsv, csv, json, sqlite, and fixed width text.
|Platform |Package Manager | Command | Out-of-box Loaders |
|-------------------|----------------------------------------|----------------------------------------------|----------------------|
|Python3.4+ |[pip3](https://visidata.org/install#pip3) | `pip3 install visidata` | Base |
|Python3.4+ |[conda](https://visidata.org/install#conda) | `conda install --channel conda-forge visidata` | Base, http, html, xls(x) |
|MacOS |[Homebrew](https://visidata.org/install#brew) | `brew install saulpw/vd/visidata` | Base, http, html, xls(x) |
|Linux (Debian/Ubuntu) |[apt](https://visidata.org/install#apt) | [full instructions](https://visidata.org/install#apt) | Base, http, html, xls(x) |
|Linux (Debian/Ubuntu) |[dpkg](https://visidata.org/install#dpkg) | [full instructions](https://visidata.org/install#dpkg) | Base, http, html, xls(x) |
|Windows |[WSL](https://visidata.org/install#wsl) | Windows is not directly supported (use WSL) | N/A |
|Python3.4+ |[github](https://visidata.org/install#git) | `pip3 install git+https://github.com/saulpw/visidata.git@stable` | Base |
Please see [/install](https://visidata.org/install) for detailed instructions, additional information, and troubleshooting.
### Usage
$ vd [] ...
$ | vd []
VisiData supports tsv, csv, xlsx, hdf5, sqlite, json and more (see the [list of supported sources](https://visidata.org/man#sources)).
Use `-f ` to force a particular filetype.
### Documentation
* [Intro to VisiData Tutorial](https://jsvine.github.io/intro-to-visidata/) by [Jeremy Singer-Vine](https://www.jsvine.com/)
* Quick reference: `Ctrl+H` within `vd` will open the [man page](https://visidata.org/man), which has a list of all commands and options.
* [keyboard list of commands](https://visidata.org/docs/kblayout)
* [/docs](https://visidata.org/docs) contains a collection of howto recipes.
### Help and Support
If you have a question, issue, or suggestion regarding VisiData, please [create an issue on Github](https://github.com/saulpw/visidata/issues) or chat with us at #visidata on [freenode.net](https://webchat.freenode.net/).
## vdtui
The core `vdtui.py` can be used to quickly create efficient terminal workflows. These have been prototyped as proof of this concept:
- [vgit](https://github.com/saulpw/vgit): a git interface
- [vsh](https://github.com/saulpw/vsh): a collection of utilities like `vping` and `vtop`.
- [vdgalcon](https://github.com/saulpw/vdgalcon): a port of the classic game [Galactic Conquest](https://www.galcon.com)
Other workflows should also be created as separate apps using vdtui. These apps can be very small and provide a lot of functionality; for example, see the included [viewtsv](bin/viewtsv).
## License
The innermost core file, `vdtui.py`, is a single-file stand-alone library that provides a solid framework for building text user interface apps. It is distributed under the MIT free software license, and freely available for inclusion in other projects.
Other VisiData components, including the main `vd` application, addons, loaders, and other code in this repository, are available for use and distribution under GPLv3.
## Credits
VisiData is conceived and developed by Saul Pwanson ``.
Anja Kefala `` maintains the documentation and packages for all platforms.
Many thanks to numerous other contributors, and to those wonderful users who provide feedback, for helping to make VisiData the awesome tool that it is.
Keywords: console tabular data spreadsheet terminal viewer textpunkcurses csv hdf5 h5 xlsx excel tsv
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable
Classifier: Environment :: Console
Classifier: Environment :: Console :: Curses
Classifier: Intended Audience :: Developers
Classifier: Intended Audience :: Science/Research
Classifier: Intended Audience :: System Administrators
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python :: 3
Classifier: Topic :: Database :: Front-Ends
Classifier: Topic :: Scientific/Engineering
Classifier: Topic :: Office/Business :: Financial :: Spreadsheet
Classifier: Topic :: Scientific/Engineering :: Visualization
Classifier: Topic :: Utilities
Requires-Python: >=3.4
visidata-1.5.2/visidata.egg-info/top_level.txt 0000777 0001750 0001750 00000000011 13416503554 022050 0 ustar kefala kefala 0000000 0000000 visidata
visidata-1.5.2/visidata.egg-info/requires.txt 0000777 0001750 0001750 00000000020 13416503554 021716 0 ustar kefala kefala 0000000 0000000 python-dateutil
visidata-1.5.2/setup.py 0000770 0001750 0001750 00000003445 13416503074 015536 0 ustar kefala kefala 0000000 0000000 #!/usr/bin/env python3
from setuptools import setup
# tox can't actually run python3 setup.py: https://github.com/tox-dev/tox/issues/96
#from visidata import __version__
__version__ = '1.5.2'
setup(name='visidata',
version=__version__,
install_requires=['python-dateutil'],
description='curses interface for exploring and arranging tabular data',
long_description=open('README.md').read(),
author='Saul Pwanson',
python_requires='>=3.4',
author_email='visidata@saul.pw',
url='https://visidata.org',
download_url='https://github.com/saulpw/visidata/tarball/' + __version__,
scripts=['bin/vd'],
py_modules = ['visidata'],
packages=['visidata', 'visidata.loaders'],
include_package_data=True,
data_files = [('share/man/man1', ['visidata/man/vd.1'])],
package_data={'visidata': ['man/vd.1'],
'visidata': ['commands.tsv']},
license='GPLv3',
classifiers=[
'Development Status :: 5 - Production/Stable',
'Environment :: Console',
'Environment :: Console :: Curses',
'Intended Audience :: Developers',
'Intended Audience :: Science/Research',
'Intended Audience :: System Administrators',
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Operating System :: OS Independent',
'Programming Language :: Python :: 3',
'Topic :: Database :: Front-Ends',
'Topic :: Scientific/Engineering',
'Topic :: Office/Business :: Financial :: Spreadsheet',
'Topic :: Scientific/Engineering :: Visualization',
'Topic :: Utilities',
],
keywords=('console tabular data spreadsheet terminal viewer textpunk'
'curses csv hdf5 h5 xlsx excel tsv'),
)
visidata-1.5.2/bin/ 0000770 0001750 0001750 00000000000 13416503554 014566 5 ustar kefala kefala 0000000 0000000 visidata-1.5.2/bin/vd 0000770 0001750 0001750 00000013056 13416503036 015125 0 ustar kefala kefala 0000000 0000000 #!/usr/bin/env python3
#
# Usage: $0 [] [ ...]
# $0 [] --play [--batch] [-w ] [-o