e.g., -1
return ops[type(node.op)](ast.literal_eval(node.operand))
else:
raise SyntaxError(node)
try:
if '#' in expr:
raise SyntaxError('no comments allowed')
return execute_ast(ast.parse(expr, mode='eval').body)
except SyntaxError:
raise SyntaxError("could not parse expression '{expr}', "
"only basic math operations are allowed (+, -, *)"
.format(expr=expr))
def force_exit(exitvalue=0):
"""force exit when thread operations could block the exit
The build command has to use some threading stuff to handle the
timeout and locks. This seems to prevent the command from
exiting, unless this hack is used.
"""
sys.stdout.flush()
sys.stderr.flush()
os._exit(exitvalue)
fdroidserver-1.1.6/fdroidserver/dscanner.py 0000644 0001750 0001750 00000041377 13576156531 021016 0 ustar hans hans 0000000 0000000 #!/usr/bin/env python3
#
# dscanner.py - part of the FDroid server tools
# Copyright (C) 2016-2017 Shawn Gustaw
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
import logging
import os
import json
import sys
from time import sleep
from argparse import ArgumentParser
from subprocess import CalledProcessError, check_output
from . import _
from . import common
from . import metadata
try:
from docker import Client
except ImportError:
logging.error(("Docker client not installed."
"Install it using pip install docker-py"))
config = None
options = None
class DockerConfig:
ALIAS = "dscanner"
CONTAINER = "dscanner/fdroidserver"
EMULATOR = "android-19"
ARCH = "armeabi-v7a"
class DockerDriver(object):
"""
Handles all the interactions with the docker container the
Android emulator runs in.
"""
class Commands:
build = ['docker', 'build', '--no-cache=false', '--pull=true',
'--quiet=false', '--rm=true', '-t',
'{0}:latest'.format(DockerConfig.CONTAINER), '.']
run = [
'docker', 'run',
'-e', '"EMULATOR={0}"'.format(DockerConfig.EMULATOR),
'-e', '"ARCH={0}"'.format(DockerConfig.ARCH),
'-d', '-P', '--name',
'{0}'.format(DockerConfig.ALIAS), '--log-driver=json-file',
DockerConfig.CONTAINER]
start = ['docker', 'start', '{0}'.format(DockerConfig.ALIAS)]
inspect = ['docker', 'inspect', '{0}'.format(DockerConfig.ALIAS)]
pm_list = 'adb shell "pm list packages"'
install_drozer = "docker exec {0} python /home/drozer/install_agent.py"
run_drozer = 'python /home/drozer/drozer.py {0}'
copy_to_container = 'docker cp "{0}" {1}:{2}'
copy_from_container = 'docker cp {0}:{1} "{2}"'
def __init__(self, init_only=False, fresh_start=False, clean_only=False):
self.container_id = None
self.ip_address = None
self.cli = Client(base_url='unix://var/run/docker.sock')
if fresh_start or clean_only:
self.clean()
if clean_only:
logging.info("Cleaned containers and quitting.")
exit(0)
self.init_docker()
if init_only:
logging.info("Initialized and quitting.")
exit(0)
def _copy_to_container(self, src_path, dest_path):
"""
Copies a file (presumed to be an apk) from src_path
to home directory on container.
"""
path = '/home/drozer/{path}.apk'.format(path=dest_path)
command = self.Commands.copy_to_container.format(src_path,
self.container_id,
path)
try:
check_output(command, shell=True)
except CalledProcessError as e:
logging.error(('Command "{command}" failed with '
'error code {code}'.format(command=command,
code=e.returncode)))
raise
def _copy_from_container(self, src_path, dest_path):
"""
Copies a file from src_path on the container to
dest_path on the host machine.
"""
command = self.Commands.copy_from_container.format(self.container_id,
src_path,
dest_path)
try:
check_output(command, shell=True)
except CalledProcessError as e:
logging.error(('Command "{command}" failed with '
'error code {code}'.format(command=command,
code=e.returncode)))
raise
logging.info("Log stored at {path}".format(path=dest_path))
def _adb_install_apk(self, apk_path):
"""
Installs an apk on the device running in the container
using adb.
"""
logging.info("Attempting to install an apk.")
exec_id = self.cli.exec_create(
self.container_id, 'adb install {0}'
.format(apk_path)
)['Id']
output = self.cli.exec_start(exec_id).decode('utf-8')
if "INSTALL_PARSE_FAILED_NO_CERTIFICATES" in output:
raise Exception('Install parse failed, no certificates')
elif "INSTALL_FAILED_ALREADY_EXISTS" in output:
logging.info("APK already installed. Skipping.")
elif "Success" not in output:
logging.error("APK didn't install properly")
return False
return True
def _adb_uninstall_apk(self, app_id):
"""
Uninstalls an application from the device running in the container
via its app_id.
"""
logging.info(
"Uninstalling {app_id} from the emulator."
.format(app_id=app_id)
)
exec_id = self.cli.exec_create(
self.container_id,
'adb uninstall {0}'.format(app_id)
)['Id']
output = self.cli.exec_start(exec_id).decode('utf-8')
if 'Success' in output:
logging.info("Successfully uninstalled.")
return True
def _verify_apk_install(self, app_id):
"""
Checks that the app_id is installed on the device running in the
container.
"""
logging.info(
"Verifying {app} is installed on the device."
.format(app=app_id)
)
exec_id = self.cli.exec_create(
self.container_id, self.Commands.pm_list
)['Id']
output = self.cli.exec_start(exec_id).decode('utf-8')
if ("Could not access the Package Manager" in output
or "device offline" in output):
logging.info("Device or package manager isn't up")
if app_id.split('_')[0] in output: # TODO: this is a temporary fix
logging.info("{app} is installed.".format(app=app_id))
return True
logging.error("APK not found in packages list on emulator.")
def _delete_file(self, path):
"""
Deletes file off the container to preserve space if scanning many apps
"""
command = "rm {path}".format(path=path)
exec_id = self.cli.exec_create(self.container_id, command)['Id']
logging.info("Deleting {path} on the container.".format(path=path))
self.cli.exec_start(exec_id)
def _install_apk(self, apk_path, app_id):
"""
Installs apk found at apk_path on the emulator. Will then
verify it installed properly by looking up its app_id in
the package manager.
"""
if not all([self.container_id, self.ip_address]):
# TODO: maybe have this fail nicely
raise Exception("Went to install apk and couldn't find container")
path = "/home/drozer/{app_id}.apk".format(app_id=app_id)
self._copy_to_container(apk_path, app_id)
self._adb_install_apk(path)
self._verify_apk_install(app_id)
self._delete_file(path)
def _install_drozer(self):
"""
Performs all the initialization of drozer within the emulator.
"""
logging.info("Attempting to install com.mwr.dz on the emulator")
logging.info("This could take a while so be patient...")
logging.info(("We need to wait for the device to boot AND"
" the package manager to come online."))
command = self.Commands.install_drozer.format(self.container_id)
try:
output = check_output(command,
shell=True).decode('utf-8')
except CalledProcessError as e:
logging.error(('Command "{command}" failed with '
'error code {code}'.format(command=command,
code=e.returncode)))
raise
if 'Installed ok' in output:
return True
def _run_drozer_scan(self, app):
"""
Runs the drozer agent which connects to the app running
on the emulator.
"""
logging.info("Running the drozer agent")
exec_id = self.cli.exec_create(
self.container_id,
self.Commands.run_drozer.format(app)
)['Id']
self.cli.exec_start(exec_id)
def _container_is_running(self):
"""
Checks whether the emulator container is running.
"""
for container in self.cli.containers():
if DockerConfig.ALIAS in container['Image']:
return True
def _docker_image_exists(self):
"""
Check whether the docker image exists already.
If this returns false we'll need to build the image
from the DockerFile.
"""
for image in self.cli.images():
for tag in image['RepoTags']:
if DockerConfig.ALIAS in tag:
return True
_image_queue = {}
def _build_docker_image(self):
"""
Builds the docker container so we can run the android emulator
inside it.
"""
logging.info("Pulling the container from docker hub")
logging.info("Image is roughly 5 GB so be patient")
logging.info("(Progress output is slow and requires a tty.)")
# we pause briefly to narrow race condition windows of opportunity
sleep(1)
is_a_tty = os.isatty(sys.stdout.fileno())
for output in self.cli.pull(
DockerConfig.CONTAINER,
stream=True,
tag="latest"):
if not is_a_tty:
# run silent, run quick
continue
try:
p = json.loads(output.decode('utf-8'))
p_id = p['id']
self._image_queue[p_id] = p
t, c, j = 1, 1, 0
for k in sorted(self._image_queue):
j += 1
v = self._image_queue[k]
vd = v['progressDetail']
t += vd['total']
c += vd['current']
msg = "\rDownloading: {0}/{1} {2}% [{3} jobs]"
msg = msg.format(c, t, int(c / t * 100), j)
sys.stdout.write(msg)
sys.stdout.flush()
except Exception:
pass
print("\nDONE!\n")
def _verify_apk_exists(self, full_apk_path):
"""
Verifies that the apk path we have is actually a file.
"""
return os.path.isfile(full_apk_path)
def init_docker(self):
"""
Perform all the initialization required before a drozer scan.
1. build the image
2. run the container
3. install drozer and enable the service within the app
"""
built = self._docker_image_exists()
if not built:
self._build_docker_image()
running = self._container_is_running()
if not running:
logging.info('Trying to run container...')
try:
check_output(self.Commands.run)
except CalledProcessError as e:
logging.error((
'Command "{command}" failed with error code {code}'
.format(command=self.Commands.run, code=e.returncode)
))
running = self._container_is_running()
if not running:
logging.info('Trying to start container...')
try:
check_output(self.Commands.start)
except CalledProcessError as e:
logging.error((
'Command "{command}" failed with error code {code}'
.format(command=self.Commands.run, code=e.returncode)
))
running = self._container_is_running()
if not running:
raise Exception("Running container not found, critical error.")
containers = self.cli.containers()
for container in containers:
if DockerConfig.ALIAS in container['Image']:
self.container_id = container['Id']
n = container['NetworkSettings']['Networks']
self.ip_address = n['bridge']['IPAddress']
break
if not self.container_id or not self.ip_address:
logging.error("No ip address or container id found.")
exit(1)
if self._verify_apk_install('com.mwr.dz'):
return
self._install_drozer()
def clean(self):
"""
Clean up all the containers made by this script.
Should be run after the drozer scan completes.
"""
for container in self.cli.containers():
if DockerConfig.ALIAS in container['Image']:
logging.info("Removing container {0}".format(container['Id']))
self.cli.remove_container(container['Id'], force=True)
def perform_drozer_scan(self, apk_path, app_id):
"""
Entrypoint for scanning an android app. Performs the following steps:
1. installs an apk on the device
2. runs a drozer scan
3. copies the report off the container
4. uninstalls the apk to save space on the device
"""
self._install_apk(apk_path, app_id)
logging.info("Running the drozer scan.")
self._run_drozer_scan(app_id)
logging.info("Scan finished. Moving the report off the container")
dest = apk_path + '.drozer'
self._copy_from_container('/tmp/drozer_report.log', dest)
self._adb_uninstall_apk(app_id)
def main():
global config, options
# Parse command line...
parser = ArgumentParser(
usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]"
)
common.setup_global_opts(parser)
parser.add_argument(
"app_id", nargs='*',
help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
parser.add_argument(
"-l", "--latest", action="store_true", default=False,
help=_("Scan only the latest version of each package"))
parser.add_argument(
"--clean-after", default=False, action='store_true',
help=_("Clean after all scans have finished"))
parser.add_argument(
"--clean-before", default=False, action='store_true',
help=_("Clean before the scans start and rebuild the container"))
parser.add_argument(
"--clean-only", default=False, action='store_true',
help=_("Clean up all containers and then exit"))
parser.add_argument(
"--init-only", default=False, action='store_true',
help=_("Prepare Drozer to run a scan"))
parser.add_argument(
"--repo-path", default="repo", action="store",
help=_("Override path for repo APKs (default: ./repo)"))
options = parser.parse_args()
config = common.read_config(options)
if not os.path.isdir(options.repo_path):
sys.stderr.write("repo-path not found: \"" + options.repo_path + "\"")
exit(1)
# Read all app and srclib metadata
allapps = metadata.read_metadata()
apps = common.read_app_args(options.app_id, allapps, True)
docker = DockerDriver(
init_only=options.init_only,
fresh_start=options.clean_before,
clean_only=options.clean_only
)
if options.clean_before:
docker.clean()
if options.clean_only:
exit(0)
for app_id, app in apps.items():
vercode = 0
if ':' in app_id:
vercode = app_id.split(':')[1]
for build in reversed(app.builds):
if build.disable:
continue
if options.latest or vercode == 0 or build.versionCode == vercode:
app.builds = [build]
break
continue
continue
for app_id, app in apps.items():
for build in app.builds:
apks = []
for f in os.listdir(options.repo_path):
n = common.get_release_filename(app, build)
if f == n:
apks.append(f)
for apk in sorted(apks):
apk_path = os.path.join(options.repo_path, apk)
docker.perform_drozer_scan(apk_path, app.id)
if options.clean_after:
docker.clean()
if __name__ == "__main__":
main()
fdroidserver-1.1.6/fdroidserver/exception.py 0000644 0001750 0001750 00000002153 13576156531 021204 0 ustar hans hans 0000000 0000000 class FDroidException(Exception):
def __init__(self, value=None, detail=None):
self.value = value
self.detail = detail
def shortened_detail(self):
if len(self.detail) < 16000:
return self.detail
return '[...]\n' + self.detail[-16000:]
def get_wikitext(self):
ret = repr(self.value) + "\n"
if self.detail:
ret += "=detail=\n"
ret += "\n" + self.shortened_detail() + "
\n"
return ret
def __str__(self):
if self.value is None:
ret = __name__
else:
ret = str(self.value)
if self.detail:
ret += "\n==== detail begin ====\n%s\n==== detail end ====" % ''.join(self.detail).strip()
return ret
class MetaDataException(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return self.value
class VCSException(FDroidException):
pass
class NoSubmodulesException(VCSException):
pass
class BuildException(FDroidException):
pass
class VerificationException(FDroidException):
pass
fdroidserver-1.1.6/fdroidserver/gpgsign.py 0000644 0001750 0001750 00000005076 13576156531 020653 0 ustar hans hans 0000000 0000000 #!/usr/bin/env python3
#
# gpgsign.py - part of the FDroid server tools
# Copyright (C) 2014, Ciaran Gultnieks, ciaran@ciarang.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
import os
import glob
from argparse import ArgumentParser
import logging
from . import _
from . import common
from .common import FDroidPopen
from .exception import FDroidException
config = None
options = None
def main():
global config, options
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options]")
common.setup_global_opts(parser)
options = parser.parse_args()
config = common.read_config(options)
repodirs = ['repo']
if config['archive_older'] != 0:
repodirs.append('archive')
for output_dir in repodirs:
if not os.path.isdir(output_dir):
raise FDroidException(_("Missing output directory") + " '" + output_dir + "'")
# Process any apks that are waiting to be signed...
for f in sorted(glob.glob(os.path.join(output_dir, '*.*'))):
if common.get_file_extension(f) == 'asc':
continue
if not common.is_repo_file(f):
continue
filename = os.path.basename(f)
sigfilename = filename + ".asc"
sigpath = os.path.join(output_dir, sigfilename)
if not os.path.exists(sigpath):
gpgargs = ['gpg', '-a',
'--output', sigpath,
'--detach-sig']
if 'gpghome' in config:
gpgargs.extend(['--homedir', config['gpghome']])
if 'gpgkey' in config:
gpgargs.extend(['--local-user', config['gpgkey']])
gpgargs.append(os.path.join(output_dir, filename))
p = FDroidPopen(gpgargs)
if p.returncode != 0:
raise FDroidException("Signing failed.")
logging.info('Signed ' + filename)
if __name__ == "__main__":
main()
fdroidserver-1.1.6/fdroidserver/import.py 0000644 0001750 0001750 00000026073 13576156531 020527 0 ustar hans hans 0000000 0000000 #!/usr/bin/env python3
#
# import.py - part of the FDroid server tools
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013-2014 Daniel Martí
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
import binascii
import os
import re
import shutil
import urllib.request
from argparse import ArgumentParser
from configparser import ConfigParser
import logging
from . import _
from . import common
from . import metadata
from .exception import FDroidException
SETTINGS_GRADLE = re.compile(r'''include\s+['"]:([^'"]*)['"]''')
# Get the repo type and address from the given web page. The page is scanned
# in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and
# when one of these is found it's assumed that's the information we want.
# Returns repotype, address, or None, reason
def getrepofrompage(url):
req = urllib.request.urlopen(url)
if req.getcode() != 200:
return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
page = req.read().decode(req.headers.get_content_charset())
# Works for BitBucket
m = re.search('data-fetch-url="(.*)"', page)
if m is not None:
repo = m.group(1)
if repo.endswith('.git'):
return ('git', repo)
return ('hg', repo)
# Works for BitBucket (obsolete)
index = page.find('hg clone')
if index != -1:
repotype = 'hg'
repo = page[index + 9:]
index = repo.find('<')
if index == -1:
return (None, _("Error while getting repo address"))
repo = repo[:index]
repo = repo.split('"')[0]
return (repotype, repo)
# Works for BitBucket (obsolete)
index = page.find('git clone')
if index != -1:
repotype = 'git'
repo = page[index + 10:]
index = repo.find('<')
if index == -1:
return (None, _("Error while getting repo address"))
repo = repo[:index]
repo = repo.split('"')[0]
return (repotype, repo)
return (None, _("No information found.") + page)
config = None
options = None
def get_metadata_from_url(app, url):
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info(_("Creating temporary directory"))
os.makedirs(tmp_dir)
# Figure out what kind of project it is...
projecttype = None
app.WebSite = url # by default, we might override it
if url.startswith('git://'):
projecttype = 'git'
repo = url
repotype = 'git'
app.SourceCode = ""
app.WebSite = ""
elif url.startswith('https://github.com'):
projecttype = 'github'
repo = url
repotype = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
app.WebSite = ""
elif url.startswith('https://gitlab.com/'):
projecttype = 'gitlab'
# git can be fussy with gitlab URLs unless they end in .git
if url.endswith('.git'):
url = url[:-4]
repo = url + '.git'
repotype = 'git'
app.WebSite = url
app.SourceCode = url + '/tree/HEAD'
app.IssueTracker = url + '/issues'
elif url.startswith('https://notabug.org/'):
projecttype = 'notabug'
if url.endswith('.git'):
url = url[:-4]
repo = url + '.git'
repotype = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
app.WebSite = ""
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
projecttype = 'bitbucket'
app.SourceCode = url + '/src'
app.IssueTracker = url + '/issues'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(url)
if not repotype:
raise FDroidException("Unable to determine vcs type. " + repo)
elif url.startswith('https://') and url.endswith('.git'):
projecttype = 'git'
repo = url
repotype = 'git'
app.SourceCode = ""
app.WebSite = ""
if not projecttype:
raise FDroidException("Unable to determine the project type. "
+ "The URL you supplied was not in one of the supported formats. "
+ "Please consult the manual for a list of supported formats, "
+ "and supply one of those.")
# Ensure we have a sensible-looking repo address at this point. If not, we
# might have got a page format we weren't expecting. (Note that we
# specifically don't want git@...)
if ((repotype != 'bzr' and (not repo.startswith('http://')
and not repo.startswith('https://')
and not repo.startswith('git://')))
or ' ' in repo):
raise FDroidException("Repo address '{0}' does not seem to be valid".format(repo))
# Get a copy of the source so we can extract some info...
logging.info('Getting source from ' + repotype + ' repo at ' + repo)
build_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(build_dir):
shutil.rmtree(build_dir)
vcs = common.getvcs(repotype, repo, build_dir)
vcs.gotorevision(options.rev)
root_dir = get_subdir(build_dir)
app.RepoType = repotype
app.Repo = repo
return root_dir, build_dir
config = None
options = None
def get_subdir(build_dir):
if options.subdir:
return os.path.join(build_dir, options.subdir)
settings_gradle = os.path.join(build_dir, 'settings.gradle')
if os.path.exists(settings_gradle):
with open(settings_gradle) as fp:
m = SETTINGS_GRADLE.search(fp.read())
if m:
return os.path.join(build_dir, m.group(1))
return build_dir
def main():
global config, options
# Parse command line...
parser = ArgumentParser()
common.setup_global_opts(parser)
parser.add_argument("-u", "--url", default=None,
help=_("Project URL to import from."))
parser.add_argument("-s", "--subdir", default=None,
help=_("Path to main Android project subdirectory, if not in root."))
parser.add_argument("-c", "--categories", default=None,
help=_("Comma separated list of categories."))
parser.add_argument("-l", "--license", default=None,
help=_("Overall license of the project."))
parser.add_argument("--rev", default=None,
help=_("Allows a different revision (or git branch) to be specified for the initial import"))
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
metadata.warnings_action = options.W
config = common.read_config(options)
apps = metadata.read_metadata()
app = metadata.App()
app.UpdateCheckMode = "Tags"
root_dir = None
build_dir = None
local_metadata_files = common.get_local_metadata_files()
if local_metadata_files != []:
raise FDroidException(_("This repo already has local metadata: %s") % local_metadata_files[0])
build = metadata.Build()
if options.url is None and os.path.isdir('.git'):
app.AutoName = os.path.basename(os.getcwd())
app.RepoType = 'git'
root_dir = get_subdir(os.getcwd())
if os.path.exists('build.gradle'):
build.gradle = ['yes']
import git
repo = git.repo.Repo(root_dir) # git repo
for remote in git.Remote.iter_items(repo):
if remote.name == 'origin':
url = repo.remotes.origin.url
if url.startswith('https://git'): # github, gitlab
app.SourceCode = url.rstrip('.git')
app.Repo = url
break
# repo.head.commit.binsha is a bytearray stored in a str
build.commit = binascii.hexlify(bytearray(repo.head.commit.binsha))
write_local_file = True
elif options.url:
root_dir, build_dir = get_metadata_from_url(app, options.url)
build.commit = '?'
build.disable = 'Generated by import.py - check/set version fields and commit id'
write_local_file = False
else:
raise FDroidException("Specify project url.")
# Extract some information...
paths = common.manifest_paths(root_dir, [])
if paths:
versionName, versionCode, package = common.parse_androidmanifests(paths, app)
if not package:
raise FDroidException(_("Couldn't find package ID"))
if not versionName:
logging.warn(_("Couldn't find latest version name"))
if not versionCode:
logging.warn(_("Couldn't find latest version code"))
else:
spec = os.path.join(root_dir, 'buildozer.spec')
if os.path.exists(spec):
defaults = {'orientation': 'landscape', 'icon': '',
'permissions': '', 'android.api': "18"}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
package = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
versionName = bconfig.get('app', 'version')
versionCode = None
else:
raise FDroidException(_("No android or kivy project could be found. Specify --subdir?"))
# Make sure it's actually new...
if package in apps:
raise FDroidException("Package " + package + " already exists")
# Create a build line...
build.versionName = versionName or 'Unknown'
build.versionCode = versionCode or '0' # TODO heinous but this is still a str
if options.subdir:
build.subdir = options.subdir
if options.license:
app.License = options.license
if options.categories:
app.Categories = options.categories.split(',')
if os.path.exists(os.path.join(root_dir, 'jni')):
build.buildjni = ['yes']
if os.path.exists(os.path.join(root_dir, 'build.gradle')):
build.gradle = ['yes']
metadata.post_metadata_parse(app)
app.builds.append(build)
if write_local_file:
metadata.write_metadata('.fdroid.yml', app)
else:
# Keep the repo directory to save bandwidth...
if not os.path.exists('build'):
os.mkdir('build')
if build_dir is not None:
shutil.move(build_dir, os.path.join('build', package))
with open('build/.fdroidvcs-' + package, 'w') as f:
f.write(app.RepoType + ' ' + app.Repo)
metadatapath = os.path.join('metadata', package + '.yml')
metadata.write_metadata(metadatapath, app)
logging.info("Wrote " + metadatapath)
if __name__ == "__main__":
main()
fdroidserver-1.1.6/fdroidserver/index.py 0000644 0001750 0001750 00000076732 13576156546 020341 0 ustar hans hans 0000000 0000000 #!/usr/bin/env python3
#
# update.py - part of the FDroid server tools
# Copyright (C) 2017, Torsten Grote
# Copyright (C) 2016, Blue Jay Wireless
# Copyright (C) 2014-2016, Hans-Christoph Steiner
# Copyright (C) 2010-2015, Ciaran Gultnieks
# Copyright (C) 2013-2014, Daniel Martí
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
import collections
import copy
import json
import logging
import os
import re
import shutil
import tempfile
import urllib.parse
import zipfile
import calendar
from binascii import hexlify, unhexlify
from datetime import datetime, timezone
from xml.dom.minidom import Document
from . import _
from . import common
from . import metadata
from . import net
from . import signindex
from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
def make(apps, sortedids, apks, repodir, archive):
"""Generate the repo index files.
This requires properly initialized options and config objects.
:param apps: fully populated apps list
:param sortedids: app package IDs, sorted
:param apks: full populated apks list
:param repodir: the repo directory
:param archive: True if this is the archive repo, False if it's the
main one.
"""
from fdroidserver.update import METADATA_VERSION
def _resolve_description_link(appid):
if appid in apps:
return "fdroid.app:" + appid, apps[appid].Name
raise MetaDataException("Cannot resolve app id " + appid)
if not common.options.nosign:
common.assert_config_keystore(common.config)
repodict = collections.OrderedDict()
repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc)
repodict['version'] = METADATA_VERSION
if common.config['repo_maxage'] != 0:
repodict['maxage'] = common.config['repo_maxage']
if archive:
repodict['name'] = common.config['archive_name']
repodict['icon'] = os.path.basename(common.config['archive_icon'])
repodict['address'] = common.config['archive_url']
repodict['description'] = common.config['archive_description']
urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
else:
repodict['name'] = common.config['repo_name']
repodict['icon'] = os.path.basename(common.config['repo_icon'])
repodict['address'] = common.config['repo_url']
repodict['description'] = common.config['repo_description']
urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
mirrorcheckfailed = False
mirrors = []
for mirror in sorted(common.config.get('mirrors', [])):
base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror)
mirrorcheckfailed = True
# must end with / or urljoin strips a whole path segment
if mirror.endswith('/'):
mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
else:
mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
for mirror in common.config.get('servergitmirrors', []):
for url in get_mirror_service_urls(mirror):
mirrors.append(url + '/' + repodir)
if mirrorcheckfailed:
raise FDroidException(_("Malformed repository mirrors."))
if mirrors:
repodict['mirrors'] = mirrors
appsWithPackages = collections.OrderedDict()
for packageName in sortedids:
app = apps[packageName]
if app['Disabled']:
continue
# only include apps with packages
for apk in apks:
if apk['packageName'] == packageName:
newapp = copy.copy(app) # update wiki needs unmodified description
newapp['Description'] = metadata.description_html(app['Description'],
_resolve_description_link)
appsWithPackages[packageName] = newapp
break
requestsdict = collections.OrderedDict()
for command in ('install', 'uninstall'):
packageNames = []
key = command + '_list'
if key in common.config:
if isinstance(common.config[key], str):
packageNames = [common.config[key]]
elif all(isinstance(item, str) for item in common.config[key]):
packageNames = common.config[key]
else:
raise TypeError(_('only accepts strings, lists, and tuples'))
requestsdict[command] = packageNames
fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
fdroid_signing_key_fingerprints)
make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
fdroid_signing_key_fingerprints)
def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
def _index_encoder_default(obj):
if isinstance(obj, set):
return sorted(list(obj))
if isinstance(obj, datetime):
# Java prefers milliseconds
# we also need to accound for time zone/daylight saving time
return int(calendar.timegm(obj.timetuple()) * 1000)
if isinstance(obj, dict):
d = collections.OrderedDict()
for key in sorted(obj.keys()):
d[key] = obj[key]
return d
raise TypeError(repr(obj) + " is not JSON serializable")
output = collections.OrderedDict()
output['repo'] = repodict
output['requests'] = requestsdict
# establish sort order of the index
v1_sort_packages(packages, fdroid_signing_key_fingerprints)
appslist = []
output['apps'] = appslist
for packageName, appdict in apps.items():
d = collections.OrderedDict()
appslist.append(d)
for k, v in sorted(appdict.items()):
if not v:
continue
if k in ('builds', 'comments', 'metadatapath',
'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
'Provides', 'Repo', 'RepoType', 'RequiresRoot',
'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
continue
# name things after the App class fields in fdroidclient
if k == 'id':
k = 'packageName'
elif k == 'CurrentVersionCode': # TODO make SuggestedVersionCode the canonical name
k = 'suggestedVersionCode'
elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name
k = 'suggestedVersionName'
elif k == 'AutoName':
if 'Name' not in apps[packageName]:
d['name'] = v
continue
else:
k = k[:1].lower() + k[1:]
d[k] = v
# establish sort order in localized dicts
for app in output['apps']:
localized = app.get('localized')
if localized:
lordered = collections.OrderedDict()
for lkey, lvalue in sorted(localized.items()):
lordered[lkey] = collections.OrderedDict()
for ikey, iname in sorted(lvalue.items()):
lordered[lkey][ikey] = iname
app['localized'] = lordered
output_packages = collections.OrderedDict()
output['packages'] = output_packages
for package in packages:
packageName = package['packageName']
if packageName not in apps:
logging.info(_('Ignoring package without metadata: ') + package['apkName'])
continue
if not package.get('versionName'):
app = apps[packageName]
versionCodeStr = str(package['versionCode']) # TODO build.versionCode should be int!
for build in app['builds']:
if build['versionCode'] == versionCodeStr:
versionName = build.get('versionName')
logging.info(_('Overriding blank versionName in {apkfilename} from metadata: {version}')
.format(apkfilename=package['apkName'], version=versionName))
package['versionName'] = versionName
break
if packageName in output_packages:
packagelist = output_packages[packageName]
else:
packagelist = []
output_packages[packageName] = packagelist
d = collections.OrderedDict()
packagelist.append(d)
for k, v in sorted(package.items()):
if not v:
continue
if k in ('icon', 'icons', 'icons_src', 'name', ):
continue
d[k] = v
json_name = 'index-v1.json'
index_file = os.path.join(repodir, json_name)
with open(index_file, 'w') as fp:
if common.options.pretty:
json.dump(output, fp, default=_index_encoder_default, indent=2)
else:
json.dump(output, fp, default=_index_encoder_default)
if common.options.nosign:
logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
else:
signindex.config = common.config
signindex.sign_index_v1(repodir, json_name)
def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
"""Sorts the supplied list to ensure a deterministic sort order for
package entries in the index file. This sort-order also expresses
installation preference to the clients.
(First in this list = first to install)
:param packages: list of packages which need to be sorted before but into index file.
"""
GROUP_DEV_SIGNED = 1
GROUP_FDROID_SIGNED = 2
GROUP_OTHER_SIGNED = 3
def v1_sort_keys(package):
packageName = package.get('packageName', None)
sig = package.get('signer', None)
dev_sig = common.metadata_find_developer_signature(packageName)
group = GROUP_OTHER_SIGNED
if dev_sig and dev_sig == sig:
group = GROUP_DEV_SIGNED
else:
fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
if fdroidsig and fdroidsig == sig:
group = GROUP_FDROID_SIGNED
versionCode = None
if package.get('versionCode', None):
versionCode = -int(package['versionCode'])
return(packageName, group, sig, versionCode)
packages.sort(key=v1_sort_keys)
def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
"""
aka index.jar aka index.xml
"""
doc = Document()
def addElement(name, value, doc, parent):
el = doc.createElement(name)
el.appendChild(doc.createTextNode(value))
parent.appendChild(el)
def addElementNonEmpty(name, value, doc, parent):
if not value:
return
addElement(name, value, doc, parent)
def addElementIfInApk(name, apk, key, doc, parent):
if key not in apk:
return
value = str(apk[key])
addElement(name, value, doc, parent)
def addElementCDATA(name, value, doc, parent):
el = doc.createElement(name)
el.appendChild(doc.createCDATASection(value))
parent.appendChild(el)
def addElementCheckLocalized(name, app, key, doc, parent, default=''):
'''Fill in field from metadata or localized block
For name/summary/description, they can come only from the app source,
or from a dir in fdroiddata. They can be entirely missing from the
metadata file if there is localized versions. This will fetch those
from the localized version if its not available in the metadata file.
'''
el = doc.createElement(name)
value = app.get(key)
lkey = key[:1].lower() + key[1:]
localized = app.get('localized')
if not value and localized:
for lang in ['en-US'] + [x for x in localized.keys()]:
if not lang.startswith('en'):
continue
if lang in localized:
value = localized[lang].get(lkey)
if value:
break
if not value and localized and len(localized) > 1:
lang = list(localized.keys())[0]
value = localized[lang].get(lkey)
if not value:
value = default
el.appendChild(doc.createTextNode(value))
parent.appendChild(el)
root = doc.createElement("fdroid")
doc.appendChild(root)
repoel = doc.createElement("repo")
repoel.setAttribute("name", repodict['name'])
if 'maxage' in repodict:
repoel.setAttribute("maxage", str(repodict['maxage']))
repoel.setAttribute("icon", os.path.basename(repodict['icon']))
repoel.setAttribute("url", repodict['address'])
addElement('description', repodict['description'], doc, repoel)
for mirror in repodict.get('mirrors', []):
addElement('mirror', mirror, doc, repoel)
repoel.setAttribute("version", str(repodict['version']))
repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
pubkey, repo_pubkey_fingerprint = extract_pubkey()
repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
root.appendChild(repoel)
for command in ('install', 'uninstall'):
for packageName in requestsdict[command]:
element = doc.createElement(command)
root.appendChild(element)
element.setAttribute('packageName', packageName)
for appid, appdict in apps.items():
app = metadata.App(appdict)
if app.Disabled is not None:
continue
# Get a list of the apks for this app...
apklist = []
apksbyversion = collections.defaultdict(lambda: [])
for apk in apks:
if apk.get('versionCode') and apk.get('packageName') == appid:
apksbyversion[apk['versionCode']].append(apk)
for versionCode, apksforver in apksbyversion.items():
fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
fdroid_signed_apk = None
name_match_apk = None
for x in apksforver:
if fdroidsig and x.get('signer', None) == fdroidsig:
fdroid_signed_apk = x
if common.apk_release_filename.match(x.get('apkName', '')):
name_match_apk = x
# choose which of the available versions is most
# suiteable for index v0
if fdroid_signed_apk:
apklist.append(fdroid_signed_apk)
elif name_match_apk:
apklist.append(name_match_apk)
else:
apklist.append(apksforver[0])
if len(apklist) == 0:
continue
apel = doc.createElement("application")
apel.setAttribute("id", app.id)
root.appendChild(apel)
addElement('id', app.id, doc, apel)
if app.added:
addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
if app.lastUpdated:
addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
addElementCheckLocalized('name', app, 'Name', doc, apel)
addElementCheckLocalized('summary', app, 'Summary', doc, apel)
if app.icon:
addElement('icon', app.icon, doc, apel)
addElementCheckLocalized('desc', app, 'Description', doc, apel,
'No description available
')
addElement('license', app.License, doc, apel)
if app.Categories:
addElement('categories', ','.join(app.Categories), doc, apel)
# We put the first (primary) category in LAST, which will have
# the desired effect of making clients that only understand one
# category see that one.
addElement('category', app.Categories[0], doc, apel)
addElement('web', app.WebSite, doc, apel)
addElement('source', app.SourceCode, doc, apel)
addElement('tracker', app.IssueTracker, doc, apel)
addElementNonEmpty('changelog', app.Changelog, doc, apel)
addElementNonEmpty('author', app.AuthorName, doc, apel)
addElementNonEmpty('email', app.AuthorEmail, doc, apel)
addElementNonEmpty('donate', app.Donate, doc, apel)
addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
addElementNonEmpty('flattr', app.FlattrID, doc, apel)
addElementNonEmpty('liberapay', app.LiberapayID, doc, apel)
# These elements actually refer to the current version (i.e. which
# one is recommended. They are historically mis-named, and need
# changing, but stay like this for now to support existing clients.
addElement('marketversion', app.CurrentVersion, doc, apel)
addElement('marketvercode', app.CurrentVersionCode, doc, apel)
if app.Provides:
pv = app.Provides.split(',')
addElementNonEmpty('provides', ','.join(pv), doc, apel)
if app.RequiresRoot:
addElement('requirements', 'root', doc, apel)
# Sort the apk list into version order, just so the web site
# doesn't have to do any work by default...
apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
if 'antiFeatures' in apklist[0]:
app.AntiFeatures.extend(apklist[0]['antiFeatures'])
if app.AntiFeatures:
addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
# Check for duplicates - they will make the client unhappy...
for i in range(len(apklist) - 1):
first = apklist[i]
second = apklist[i + 1]
if first['versionCode'] == second['versionCode'] \
and first['sig'] == second['sig']:
if first['hash'] == second['hash']:
raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
repodir, first['apkName'], second['apkName']))
else:
raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
repodir, first['apkName'], second['apkName']))
current_version_code = 0
current_version_file = None
for apk in apklist:
file_extension = common.get_file_extension(apk['apkName'])
# find the APK for the "Current Version"
if current_version_code < apk['versionCode']:
current_version_code = apk['versionCode']
if current_version_code < int(app.CurrentVersionCode):
current_version_file = apk['apkName']
apkel = doc.createElement("package")
apel.appendChild(apkel)
versionName = apk.get('versionName')
if not versionName:
versionCodeStr = str(apk['versionCode']) # TODO build.versionCode should be int!
for build in app.builds:
if build['versionCode'] == versionCodeStr and 'versionName' in build:
versionName = build['versionName']
break
if versionName:
addElement('version', versionName, doc, apkel)
addElement('versioncode', str(apk['versionCode']), doc, apkel)
addElement('apkname', apk['apkName'], doc, apkel)
addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
hashel = doc.createElement("hash")
hashel.setAttribute('type', 'sha256')
hashel.appendChild(doc.createTextNode(apk['hash']))
apkel.appendChild(hashel)
addElement('size', str(apk['size']), doc, apkel)
addElementIfInApk('sdkver', apk,
'minSdkVersion', doc, apkel)
addElementIfInApk('targetSdkVersion', apk,
'targetSdkVersion', doc, apkel)
addElementIfInApk('maxsdkver', apk,
'maxSdkVersion', doc, apkel)
addElementIfInApk('obbMainFile', apk,
'obbMainFile', doc, apkel)
addElementIfInApk('obbMainFileSha256', apk,
'obbMainFileSha256', doc, apkel)
addElementIfInApk('obbPatchFile', apk,
'obbPatchFile', doc, apkel)
addElementIfInApk('obbPatchFileSha256', apk,
'obbPatchFileSha256', doc, apkel)
if 'added' in apk:
addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
if file_extension == 'apk': # sig is required for APKs, but only APKs
addElement('sig', apk['sig'], doc, apkel)
old_permissions = set()
sorted_permissions = sorted(apk['uses-permission'])
for perm in sorted_permissions:
perm_name = perm[0]
if perm_name.startswith("android.permission."):
perm_name = perm_name[19:]
old_permissions.add(perm_name)
addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
for permission in sorted_permissions:
permel = doc.createElement('uses-permission')
permel.setAttribute('name', permission[0])
if permission[1] is not None:
permel.setAttribute('maxSdkVersion', '%d' % permission[1])
apkel.appendChild(permel)
for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
permel = doc.createElement('uses-permission-sdk-23')
permel.setAttribute('name', permission_sdk_23[0])
if permission_sdk_23[1] is not None:
permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23[1])
apkel.appendChild(permel)
if 'nativecode' in apk:
addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
if current_version_file is not None \
and common.config['make_current_version_link'] \
and repodir == 'repo': # only create these
namefield = common.config['current_version_name_source']
sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
if os.path.islink(apklinkname):
os.remove(apklinkname)
os.symlink(current_version_path, apklinkname)
# also symlink gpg signature, if it exists
for extension in (b'.asc', b'.sig'):
sigfile_path = current_version_path + extension
if os.path.exists(sigfile_path):
siglinkname = apklinkname + extension
if os.path.islink(siglinkname):
os.remove(siglinkname)
os.symlink(sigfile_path, siglinkname)
if common.options.pretty:
output = doc.toprettyxml(encoding='utf-8')
else:
output = doc.toxml(encoding='utf-8')
with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
f.write(output)
if 'repo_keyalias' in common.config:
if common.options.nosign:
logging.info(_("Creating unsigned index in preparation for signing"))
else:
logging.info(_("Creating signed index with this key (SHA256):"))
logging.info("%s" % repo_pubkey_fingerprint)
# Create a jar of the index...
jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
if p.returncode != 0:
raise FDroidException("Failed to create {0}".format(jar_output))
# Sign the index...
signed = os.path.join(repodir, 'index.jar')
if common.options.nosign:
# Remove old signed index if not signing
if os.path.exists(signed):
os.remove(signed)
else:
signindex.config = common.config
signindex.sign_jar(signed)
# Copy the repo icon into the repo directory...
icon_dir = os.path.join(repodir, 'icons')
iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
shutil.copyfile(common.config['repo_icon'], iconfilename)
def extract_pubkey():
"""
Extracts and returns the repository's public key from the keystore.
:return: public key in hex, repository fingerprint
"""
if 'repo_pubkey' in common.config:
pubkey = unhexlify(common.config['repo_pubkey'])
else:
env_vars = {'LC_ALL': 'C.UTF-8',
'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
'-alias', common.config['repo_keyalias'],
'-keystore', common.config['keystore'],
'-storepass:env', 'FDROID_KEY_STORE_PASS']
+ common.config['smartcardoptions'],
envs=env_vars, output=False, stderr_to_stdout=False)
if p.returncode != 0 or len(p.output) < 20:
msg = "Failed to get repo pubkey!"
if common.config['keystore'] == 'NONE':
msg += ' Is your crypto smartcard plugged in?'
raise FDroidException(msg)
pubkey = p.output
repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
return hexlify(pubkey), repo_pubkey_fingerprint
def get_mirror_service_urls(url):
'''Get direct URLs from git service for use by fdroidclient
Via 'servergitmirrors', fdroidserver can create and push a mirror
to certain well known git services like gitlab or github. This
will always use the 'master' branch since that is the default
branch in git. The files are then accessible via alternate URLs,
where they are served in their raw format via a CDN rather than
from git.
'''
if url.startswith('git@'):
url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
segments = url.split("/")
if segments[4].endswith('.git'):
segments[4] = segments[4][:-4]
hostname = segments[2]
user = segments[3]
repo = segments[4]
branch = "master"
folder = "fdroid"
urls = []
if hostname == "github.com":
# Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder"
segments[2] = "raw.githubusercontent.com"
segments.extend([branch, folder])
urls.append('/'.join(segments))
elif hostname == "gitlab.com":
# Both these Gitlab URLs will work with F-Droid, but only the first will work in the browser
# This is because the `raw` URLs are not served with the correct mime types, so any
# index.html which is put in the repo will not be rendered. Putting an index.html file in
# the repo root is a common way for to make information about the repo available to end user.
# Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
urls.append('/'.join(gitlab_pages))
# Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
gitlab_raw = segments + ['raw', branch, folder]
urls.append('/'.join(gitlab_raw))
return urls
return urls
def download_repo_index(url_str, etag=None, verify_fingerprint=True, timeout=600):
"""Downloads and verifies index file, then returns its data.
Downloads the repository index from the given :param url_str and
verifies the repository's fingerprint if :param verify_fingerprint
is not False.
:raises: VerificationException() if the repository could not be verified
:return: A tuple consisting of:
- The index in JSON format or None if the index did not change
- The new eTag as returned by the HTTP request
"""
url = urllib.parse.urlsplit(url_str)
fingerprint = None
if verify_fingerprint:
query = urllib.parse.parse_qs(url.query)
if 'fingerprint' not in query:
raise VerificationException(_("No fingerprint in URL."))
fingerprint = query['fingerprint'][0]
url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
download, new_etag = net.http_get(url.geturl(), etag, timeout)
if download is None:
return None, new_etag
with tempfile.NamedTemporaryFile() as fp:
fp.write(download)
fp.flush()
index, public_key, public_key_fingerprint = get_index_from_jar(fp.name, fingerprint)
index["repo"]["pubkey"] = hexlify(public_key).decode()
index["repo"]["fingerprint"] = public_key_fingerprint
index["apps"] = [metadata.App(app) for app in index["apps"]]
return index, new_etag
def get_index_from_jar(jarfile, fingerprint=None):
"""Returns the data, public key, and fingerprint from index-v1.jar
:raises: VerificationException() if the repository could not be verified
"""
logging.debug(_('Verifying index signature:'))
common.verify_jar_signature(jarfile)
with zipfile.ZipFile(jarfile) as jar:
public_key, public_key_fingerprint = get_public_key_from_jar(jar)
if fingerprint is not None:
if fingerprint.upper() != public_key_fingerprint:
raise VerificationException(_("The repository's fingerprint does not match."))
data = json.loads(jar.read('index-v1.json').decode())
return data, public_key, public_key_fingerprint
def get_public_key_from_jar(jar):
"""
Get the public key and its fingerprint from a JAR file.
:raises: VerificationException() if the JAR was not signed exactly once
:param jar: a zipfile.ZipFile object
:return: the public key from the jar and its fingerprint
"""
# extract certificate from jar
certs = [n for n in jar.namelist() if common.SIGNATURE_BLOCK_FILE_REGEX.match(n)]
if len(certs) < 1:
raise VerificationException(_("Found no signing certificates for repository."))
if len(certs) > 1:
raise VerificationException(_("Found multiple signing certificates for repository."))
# extract public key from certificate
public_key = common.get_certificate(jar.read(certs[0]))
public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
return public_key, public_key_fingerprint
fdroidserver-1.1.6/fdroidserver/init.py 0000644 0001750 0001750 00000027327 13576156531 020163 0 ustar hans hans 0000000 0000000 #!/usr/bin/env python3
#
# init.py - part of the FDroid server tools
# Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013-2014 Daniel Martí
# Copyright (C) 2013 Hans-Christoph Steiner
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
import glob
import os
import re
import shutil
import socket
import sys
from argparse import ArgumentParser
import logging
from . import _
from . import common
from .exception import FDroidException
config = {}
options = None
def disable_in_config(key, value):
'''write a key/value to the local config.py, then comment it out'''
with open('config.py', 'r') as f:
data = f.read()
pattern = r'\n[\s#]*' + key + r'\s*=\s*"[^"]*"'
repl = '\n#' + key + ' = "' + value + '"'
data = re.sub(pattern, repl, data)
with open('config.py', 'w') as f:
f.writelines(data)
def main():
global options, config
# Parse command line...
parser = ArgumentParser()
common.setup_global_opts(parser)
parser.add_argument("-d", "--distinguished-name", default=None,
help=_("X.509 'Distinguished Name' used when generating keys"))
parser.add_argument("--keystore", default=None,
help=_("Path to the keystore for the repo signing key"))
parser.add_argument("--repo-keyalias", default=None,
help=_("Alias of the repo signing key in the keystore"))
parser.add_argument("--android-home", default=None,
help=_("Path to the Android SDK (sometimes set in ANDROID_HOME)"))
parser.add_argument("--no-prompt", action="store_true", default=False,
help=_("Do not prompt for Android SDK path, just fail"))
options = parser.parse_args()
aapt = None
fdroiddir = os.getcwd()
test_config = dict()
examplesdir = common.get_examples_dir()
common.fill_config_defaults(test_config)
# track down where the Android SDK is, the default is to use the path set
# in ANDROID_HOME if that exists, otherwise None
if options.android_home is not None:
test_config['sdk_path'] = options.android_home
elif common.use_androguard():
pass
elif not common.test_sdk_exists(test_config):
if os.path.isfile('/usr/bin/aapt'):
# remove sdk_path and build_tools, they are not required
test_config.pop('sdk_path', None)
test_config.pop('build_tools', None)
# make sure at least aapt is found, since this can't do anything without it
test_config['aapt'] = common.find_sdk_tools_cmd('aapt')
else:
# if neither --android-home nor the default sdk_path
# exist, prompt the user using platform-specific default
default_sdk_path = '/opt/android-sdk'
if sys.platform == 'win32' or sys.platform == 'cygwin':
p = os.path.join(os.getenv('USERPROFILE'),
'AppData', 'Local', 'Android', 'android-sdk')
elif sys.platform == 'darwin':
# on OSX, Homebrew is common and has an easy path to detect
p = '/usr/local/opt/android-sdk'
else:
# if the Debian packages are installed, suggest them
p = '/usr/lib/android-sdk'
if os.path.exists(p):
default_sdk_path = p
while not options.no_prompt:
try:
s = input(_('Enter the path to the Android SDK (%s) here:\n> ') % default_sdk_path)
except KeyboardInterrupt:
print('')
sys.exit(1)
if re.match(r'^\s*$', s) is not None:
test_config['sdk_path'] = default_sdk_path
else:
test_config['sdk_path'] = s
if common.test_sdk_exists(test_config):
break
if (options.android_home is not None or not common.use_androguard()) \
and not common.test_sdk_exists(test_config):
raise FDroidException("Android SDK not found.")
if not os.path.exists('config.py'):
# 'metadata' and 'tmp' are created in fdroid
if not os.path.exists('repo'):
os.mkdir('repo')
shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir)
shutil.copyfile(os.path.join(examplesdir, 'config.py'), 'config.py')
os.chmod('config.py', 0o0600)
# If android_home is None, test_config['sdk_path'] will be used and
# "$ANDROID_HOME" may be used if the env var is set up correctly.
# If android_home is not None, the path given from the command line
# will be directly written in the config.
if 'sdk_path' in test_config:
common.write_to_config(test_config, 'sdk_path', options.android_home)
else:
logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
logging.info('Try running `fdroid init` in an empty directory.')
raise FDroidException('Repository already exists.')
if common.use_androguard():
pass
elif 'aapt' not in test_config or not os.path.isfile(test_config['aapt']):
# try to find a working aapt, in all the recent possible paths
build_tools = os.path.join(test_config['sdk_path'], 'build-tools')
aaptdirs = []
aaptdirs.append(os.path.join(build_tools, test_config['build_tools']))
aaptdirs.append(build_tools)
for f in os.listdir(build_tools):
if os.path.isdir(os.path.join(build_tools, f)):
aaptdirs.append(os.path.join(build_tools, f))
for d in sorted(aaptdirs, reverse=True):
if os.path.isfile(os.path.join(d, 'aapt')):
aapt = os.path.join(d, 'aapt')
break
if aapt and os.path.isfile(aapt):
dirname = os.path.basename(os.path.dirname(aapt))
if dirname == 'build-tools':
# this is the old layout, before versioned build-tools
test_config['build_tools'] = ''
else:
test_config['build_tools'] = dirname
common.write_to_config(test_config, 'build_tools')
common.ensure_build_tools_exists(test_config)
# now that we have a local config.py, read configuration...
config = common.read_config(options)
# the NDK is optional and there may be multiple versions of it, so it's
# left for the user to configure
# find or generate the keystore for the repo signing key. First try the
# path written in the default config.py. Then check if the user has
# specified a path from the command line, which will trump all others.
# Otherwise, create ~/.local/share/fdroidserver and stick it in there. If
# keystore is set to NONE, that means that Java will look for keys in a
# Hardware Security Module aka Smartcard.
keystore = config['keystore']
if options.keystore:
keystore = os.path.abspath(options.keystore)
if options.keystore == 'NONE':
keystore = options.keystore
else:
keystore = os.path.abspath(options.keystore)
if not os.path.exists(keystore):
logging.info('"' + keystore
+ '" does not exist, creating a new keystore there.')
common.write_to_config(test_config, 'keystore', keystore)
repo_keyalias = None
keydname = None
if options.repo_keyalias:
repo_keyalias = options.repo_keyalias
common.write_to_config(test_config, 'repo_keyalias', repo_keyalias)
if options.distinguished_name:
keydname = options.distinguished_name
common.write_to_config(test_config, 'keydname', keydname)
if keystore == 'NONE': # we're using a smartcard
common.write_to_config(test_config, 'repo_keyalias', '1') # seems to be the default
disable_in_config('keypass', 'never used with smartcard')
common.write_to_config(test_config, 'smartcardoptions',
('-storetype PKCS11 -providerName SunPKCS11-OpenSC '
+ '-providerClass sun.security.pkcs11.SunPKCS11 '
+ '-providerArg opensc-fdroid.cfg'))
# find opensc-pkcs11.so
if not os.path.exists('opensc-fdroid.cfg'):
if os.path.exists('/usr/lib/opensc-pkcs11.so'):
opensc_so = '/usr/lib/opensc-pkcs11.so'
elif os.path.exists('/usr/lib64/opensc-pkcs11.so'):
opensc_so = '/usr/lib64/opensc-pkcs11.so'
else:
files = glob.glob('/usr/lib/' + os.uname()[4] + '-*-gnu/opensc-pkcs11.so')
if len(files) > 0:
opensc_so = files[0]
else:
opensc_so = '/usr/lib/opensc-pkcs11.so'
logging.warn('No OpenSC PKCS#11 module found, '
+ 'install OpenSC then edit "opensc-fdroid.cfg"!')
with open(os.path.join(examplesdir, 'opensc-fdroid.cfg'), 'r') as f:
opensc_fdroid = f.read()
opensc_fdroid = re.sub('^library.*', 'library = ' + opensc_so, opensc_fdroid,
flags=re.MULTILINE)
with open('opensc-fdroid.cfg', 'w') as f:
f.write(opensc_fdroid)
elif os.path.exists(keystore):
to_set = ['keystorepass', 'keypass', 'repo_keyalias', 'keydname']
if repo_keyalias:
to_set.remove('repo_keyalias')
if keydname:
to_set.remove('keydname')
logging.warning('\n' + _('Using existing keystore "{path}"').format(path=keystore)
+ '\n' + _('Now set these in config.py:') + ' '
+ ', '.join(to_set) + '\n')
else:
password = common.genpassword()
c = dict(test_config)
c['keystorepass'] = password
c['keypass'] = password
c['repo_keyalias'] = socket.getfqdn()
c['keydname'] = 'CN=' + c['repo_keyalias'] + ', OU=F-Droid'
common.write_to_config(test_config, 'keystorepass', password)
common.write_to_config(test_config, 'keypass', password)
common.write_to_config(test_config, 'repo_keyalias', c['repo_keyalias'])
common.write_to_config(test_config, 'keydname', c['keydname'])
common.genkeystore(c)
msg = '\n'
msg += _('Built repo based in "%s" with this config:') % fdroiddir
msg += '\n\n Android SDK:\t\t\t' + config['sdk_path']
if aapt:
msg += '\n Android SDK Build Tools:\t' + os.path.dirname(aapt)
msg += '\n Android NDK r12b (optional):\t$ANDROID_NDK'
msg += '\n ' + _('Keystore for signing key:\t') + keystore
if repo_keyalias is not None:
msg += '\n Alias for key in store:\t' + repo_keyalias
msg += '\n\n' + '''To complete the setup, add your APKs to "%s"
then run "fdroid update -c; fdroid update". You might also want to edit
"config.py" to set the URL, repo name, and more. You should also set up
a signing key (a temporary one might have been automatically generated).
For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo
and https://f-droid.org/docs/Signing_Process''' % os.path.join(fdroiddir, 'repo')
logging.info(msg)
fdroidserver-1.1.6/fdroidserver/install.py 0000644 0001750 0001750 00000010357 13576156531 020661 0 ustar hans hans 0000000 0000000 #!/usr/bin/env python3
#
# install.py - part of the FDroid server tools
# Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013-2014 Daniel Martí
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
import sys
import os
import glob
from argparse import ArgumentParser
import logging
from . import _
from . import common
from .common import SdkToolsPopen
from .exception import FDroidException
options = None
config = None
def devices():
p = SdkToolsPopen(['adb', "devices"])
if p.returncode != 0:
raise FDroidException("An error occured when finding devices: %s" % p.output)
lines = [l for l in p.output.splitlines() if not l.startswith('* ')]
if len(lines) < 3:
return []
lines = lines[1:-1]
return [l.split()[0] for l in lines]
def main():
global options, config
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
common.setup_global_opts(parser)
parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
parser.add_argument("-a", "--all", action="store_true", default=False,
help=_("Install all signed applications available"))
options = parser.parse_args()
if not options.appid and not options.all:
parser.error(_("option %s: If you really want to install all the signed apps, use --all") % "all")
config = common.read_config(options)
output_dir = 'repo'
if not os.path.isdir(output_dir):
logging.info(_("No signed output directory - nothing to do"))
sys.exit(0)
if options.appid:
vercodes = common.read_pkg_args(options.appid, True)
apks = {appid: None for appid in vercodes}
# Get the signed apk with the highest vercode
for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))):
try:
appid, vercode = common.publishednameinfo(apkfile)
except FDroidException:
continue
if appid not in apks:
continue
if vercodes[appid] and vercode not in vercodes[appid]:
continue
apks[appid] = apkfile
for appid, apk in apks.items():
if not apk:
raise FDroidException(_("No signed apk available for %s") % appid)
else:
apks = {common.publishednameinfo(apkfile)[0]: apkfile for apkfile in
sorted(glob.glob(os.path.join(output_dir, '*.apk')))}
for appid, apk in apks.items():
# Get device list each time to avoid device not found errors
devs = devices()
if not devs:
raise FDroidException(_("No attached devices found"))
logging.info(_("Installing %s...") % apk)
for dev in devs:
logging.info(_("Installing '{apkfilename}' on {dev}...").format(apkfilename=apk, dev=dev))
p = SdkToolsPopen(['adb', "-s", dev, "install", apk])
fail = ""
for line in p.output.splitlines():
if line.startswith("Failure"):
fail = line[9:-1]
if not fail:
continue
if fail == "INSTALL_FAILED_ALREADY_EXISTS":
logging.warn(_("'{apkfilename}' is already installed on {dev}.")
.format(apkfilename=apk, dev=dev))
else:
raise FDroidException(_("Failed to install '{apkfilename}' on {dev}: {error}")
.format(apkfilename=apk, dev=dev, error=fail))
logging.info('\n' + _('Finished'))
if __name__ == "__main__":
main()
fdroidserver-1.1.6/fdroidserver/lint.py 0000644 0001750 0001750 00000066361 13576156546 020175 0 ustar hans hans 0000000 0000000 #!/usr/bin/env python3
#
# lint.py - part of the FDroid server tool
# Copyright (C) 2013-2014 Daniel Martí
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See th
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public Licen
# along with this program. If not, see .
from argparse import ArgumentParser
import glob
import os
import re
import sys
import urllib.parse
from . import _
from . import common
from . import metadata
from . import rewritemeta
config = None
options = None
def enforce_https(domain):
return (re.compile(r'^[^h][^t][^t][^p][^s]://[^/]*' + re.escape(domain) + r'(/.*)?', re.IGNORECASE),
domain + " URLs should always use https://")
https_enforcings = [
enforce_https('github.com'),
enforce_https('gitlab.com'),
enforce_https('bitbucket.org'),
enforce_https('apache.org'),
enforce_https('google.com'),
enforce_https('git.code.sf.net'),
enforce_https('svn.code.sf.net'),
enforce_https('anongit.kde.org'),
enforce_https('savannah.nongnu.org'),
enforce_https('git.savannah.nongnu.org'),
enforce_https('download.savannah.nongnu.org'),
enforce_https('savannah.gnu.org'),
enforce_https('git.savannah.gnu.org'),
enforce_https('download.savannah.gnu.org'),
enforce_https('github.io'),
enforce_https('gitlab.io'),
enforce_https('githubusercontent.com'),
]
def forbid_shortener(domain):
return (re.compile(r'https?://[^/]*' + re.escape(domain) + r'/.*'),
_("URL shorteners should not be used"))
http_url_shorteners = [
forbid_shortener('1url.com'),
forbid_shortener('adf.ly'),
forbid_shortener('bc.vc'),
forbid_shortener('bit.do'),
forbid_shortener('bit.ly'),
forbid_shortener('bitly.com'),
forbid_shortener('budurl.com'),
forbid_shortener('buzurl.com'),
forbid_shortener('cli.gs'),
forbid_shortener('cur.lv'),
forbid_shortener('cutt.us'),
forbid_shortener('db.tt'),
forbid_shortener('filoops.info'),
forbid_shortener('goo.gl'),
forbid_shortener('is.gd'),
forbid_shortener('ity.im'),
forbid_shortener('j.mp'),
forbid_shortener('l.gg'),
forbid_shortener('lnkd.in'),
forbid_shortener('moourl.com'),
forbid_shortener('ow.ly'),
forbid_shortener('para.pt'),
forbid_shortener('po.st'),
forbid_shortener('q.gs'),
forbid_shortener('qr.ae'),
forbid_shortener('qr.net'),
forbid_shortener('rdlnk.com'),
forbid_shortener('scrnch.me'),
forbid_shortener('short.nr'),
forbid_shortener('sn.im'),
forbid_shortener('snipurl.com'),
forbid_shortener('su.pr'),
forbid_shortener('t.co'),
forbid_shortener('tiny.cc'),
forbid_shortener('tinyarrows.com'),
forbid_shortener('tinyurl.com'),
forbid_shortener('tr.im'),
forbid_shortener('tweez.me'),
forbid_shortener('twitthis.com'),
forbid_shortener('twurl.nl'),
forbid_shortener('tyn.ee'),
forbid_shortener('u.bb'),
forbid_shortener('u.to'),
forbid_shortener('ur1.ca'),
forbid_shortener('urlof.site'),
forbid_shortener('v.gd'),
forbid_shortener('vzturl.com'),
forbid_shortener('x.co'),
forbid_shortener('xrl.us'),
forbid_shortener('yourls.org'),
forbid_shortener('zip.net'),
forbid_shortener('✩.ws'),
forbid_shortener('➡.ws'),
]
http_checks = https_enforcings + http_url_shorteners + [
(re.compile(r'.*github\.com/[^/]+/[^/]+\.git'),
_("Appending .git is not necessary")),
(re.compile(r'.*://[^/]*(github|gitlab|bitbucket|rawgit)[^/]*/([^/]+/){1,3}master'),
_("Use /HEAD instead of /master to point at a file in the default branch")),
]
regex_checks = {
'WebSite': http_checks,
'SourceCode': http_checks,
'Repo': https_enforcings,
'UpdateCheckMode': https_enforcings,
'IssueTracker': http_checks + [
(re.compile(r'.*github\.com/[^/]+/[^/]+/*$'),
_("/issues is missing")),
(re.compile(r'.*gitlab\.com/[^/]+/[^/]+/*$'),
_("/issues is missing")),
],
'Donate': http_checks + [
(re.compile(r'.*flattr\.com'),
_("Flattr donation methods belong in the FlattrID flag")),
(re.compile(r'.*liberapay\.com'),
_("Liberapay donation methods belong in the LiberapayID flag")),
],
'Changelog': http_checks,
'Author Name': [
(re.compile(r'^\s'),
_("Unnecessary leading space")),
(re.compile(r'.*\s$'),
_("Unnecessary trailing space")),
],
'Summary': [
(re.compile(r'.*\b(free software|open source)\b.*', re.IGNORECASE),
_("No need to specify that the app is Free Software")),
(re.compile(r'.*((your|for).*android|android.*(app|device|client|port|version))', re.IGNORECASE),
_("No need to specify that the app is for Android")),
(re.compile(r'.*[a-z0-9][.!?]( |$)'),
_("Punctuation should be avoided")),
(re.compile(r'^\s'),
_("Unnecessary leading space")),
(re.compile(r'.*\s$'),
_("Unnecessary trailing space")),
],
'Description': https_enforcings + http_url_shorteners + [
(re.compile(r'\s*[*#][^ .]'),
_("Invalid bulleted list")),
(re.compile(r'https://f-droid.org/[a-z][a-z](_[A-Za-z]{2,4})?/'),
_("Locale included in f-droid.org URL")),
(re.compile(r'^\s'),
_("Unnecessary leading space")),
(re.compile(r'.*\s$'),
_("Unnecessary trailing space")),
(re.compile(r'.*<(applet|base|body|button|embed|form|head|html|iframe|img|input|link|object|picture|script|source|style|svg|video).*', re.IGNORECASE),
_("Forbidden HTML tags")),
(re.compile(r'''.*\s+src=["']javascript:.*'''),
_("Javascript in HTML src attributes")),
],
}
locale_pattern = re.compile(r'^[a-z]{2,3}(-[A-Z][A-Z])?$')
def check_regexes(app):
for f, checks in regex_checks.items():
for m, r in checks:
v = app.get(f)
t = metadata.fieldtype(f)
if t == metadata.TYPE_MULTILINE:
for l in v.splitlines():
if m.match(l):
yield "%s at line '%s': %s" % (f, l, r)
else:
if v is None:
continue
if m.match(v):
yield "%s '%s': %s" % (f, v, r)
def get_lastbuild(builds):
lowest_vercode = -1
lastbuild = None
for build in builds:
if not build.disable:
vercode = int(build.versionCode)
if lowest_vercode == -1 or vercode < lowest_vercode:
lowest_vercode = vercode
if not lastbuild or int(build.versionCode) > int(lastbuild.versionCode):
lastbuild = build
return lastbuild
def check_update_check_data_url(app):
"""UpdateCheckData must have a valid HTTPS URL to protect checkupdates runs
"""
if app.UpdateCheckData:
urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')
for url in (urlcode, urlver):
if url != '.':
parsed = urllib.parse.urlparse(url)
if not parsed.scheme or not parsed.netloc:
yield _('UpdateCheckData not a valid URL: {url}').format(url=url)
if parsed.scheme != 'https':
yield _('UpdateCheckData must use HTTPS URL: {url}').format(url=url)
def check_vercode_operation(app):
if app.VercodeOperation and not common.VERCODE_OPERATION_RE.match(app.VercodeOperation):
yield _('Invalid VercodeOperation: {field}').format(field=app.VercodeOperation)
def check_ucm_tags(app):
lastbuild = get_lastbuild(app.builds)
if (lastbuild is not None
and lastbuild.commit
and app.UpdateCheckMode == 'RepoManifest'
and not lastbuild.commit.startswith('unknown')
and lastbuild.versionCode == app.CurrentVersionCode
and not lastbuild.forcevercode
and any(s in lastbuild.commit for s in '.,_-/')):
yield _("Last used commit '{commit}' looks like a tag, but Update Check Mode is '{ucm}'")\
.format(commit=lastbuild.commit, ucm=app.UpdateCheckMode)
def check_char_limits(app):
limits = config['char_limits']
if len(app.Summary) > limits['summary']:
yield _("Summary of length {length} is over the {limit} char limit")\
.format(length=len(app.Summary), limit=limits['summary'])
if len(app.Description) > limits['description']:
yield _("Description of length {length} is over the {limit} char limit")\
.format(length=len(app.Description), limit=limits['description'])
def check_old_links(app):
usual_sites = [
'github.com',
'gitlab.com',
'bitbucket.org',
]
old_sites = [
'gitorious.org',
'code.google.com',
]
if any(s in app.Repo for s in usual_sites):
for f in ['WebSite', 'SourceCode', 'IssueTracker', 'Changelog']:
v = app.get(f)
if any(s in v for s in old_sites):
yield _("App is in '{repo}' but has a link to {url}")\
.format(repo=app.Repo, url=v)
def check_useless_fields(app):
if app.UpdateCheckName == app.id:
yield _("Update Check Name is set to the known app id - it can be removed")
filling_ucms = re.compile(r'^(Tags.*|RepoManifest.*)')
def check_checkupdates_ran(app):
if filling_ucms.match(app.UpdateCheckMode):
if not app.AutoName and not app.CurrentVersion and app.CurrentVersionCode == '0':
yield _("UCM is set but it looks like checkupdates hasn't been run yet")
def check_empty_fields(app):
if not app.Categories:
yield _("Categories are not set")
all_categories = set([
"Connectivity",
"Development",
"Games",
"Graphics",
"Internet",
"Money",
"Multimedia",
"Navigation",
"Phone & SMS",
"Reading",
"Science & Education",
"Security",
"Sports & Health",
"System",
"Theming",
"Time",
"Writing",
])
def check_categories(app):
for categ in app.Categories:
if categ not in all_categories:
yield _("Categories '%s' is not valid" % categ)
def check_duplicates(app):
if app.Name and app.Name == app.AutoName:
yield _("Name '%s' is just the auto name - remove it") % app.Name
links_seen = set()
for f in ['Source Code', 'Web Site', 'Issue Tracker', 'Changelog']:
v = app.get(f)
if not v:
continue
v = v.lower()
if v in links_seen:
yield _("Duplicate link in '{field}': {url}").format(field=f, url=v)
else:
links_seen.add(v)
name = app.Name or app.AutoName
if app.Summary and name:
if app.Summary.lower() == name.lower():
yield _("Summary '%s' is just the app's name") % app.Summary
if app.Summary and app.Description and len(app.Description) == 1:
if app.Summary.lower() == app.Description[0].lower():
yield _("Description '%s' is just the app's summary") % app.Summary
seenlines = set()
for l in app.Description.splitlines():
if len(l) < 1:
continue
if l in seenlines:
yield _("Description has a duplicate line")
seenlines.add(l)
desc_url = re.compile(r'(^|[^[])\[([^ ]+)( |\]|$)')
def check_mediawiki_links(app):
wholedesc = ' '.join(app.Description)
for um in desc_url.finditer(wholedesc):
url = um.group(1)
for m, r in http_checks:
if m.match(url):
yield _("URL {url} in Description: {error}").format(url=url, error=r)
def check_bulleted_lists(app):
validchars = ['*', '#']
lchar = ''
lcount = 0
for l in app.Description.splitlines():
if len(l) < 1:
lcount = 0
continue
if l[0] == lchar and l[1] == ' ':
lcount += 1
if lcount > 2 and lchar not in validchars:
yield _("Description has a list (%s) but it isn't bulleted (*) nor numbered (#)") % lchar
break
else:
lchar = l[0]
lcount = 1
def check_builds(app):
supported_flags = set(metadata.build_flags)
# needed for YAML and JSON
for build in app.builds:
if build.disable:
if build.disable.startswith('Generated by import.py'):
yield _("Build generated by `fdroid import` - remove disable line once ready")
continue
for s in ['master', 'origin', 'HEAD', 'default', 'trunk']:
if build.commit and build.commit.startswith(s):
yield _("Branch '{branch}' used as commit in build '{versionName}'")\
.format(branch=s, versionName=build.versionName)
for srclib in build.srclibs:
if '@' in srclib:
ref = srclib.split('@')[1].split('/')[0]
if ref.startswith(s):
yield _("Branch '{branch}' used as commit in srclib '{srclib}'")\
.format(branch=s, srclib=srclib)
else:
yield _('srclibs missing name and/or @') + ' (srclibs: ' + srclib + ')'
for key in build.keys():
if key not in supported_flags:
yield _('%s is not an accepted build field') % key
def check_files_dir(app):
dir_path = os.path.join('metadata', app.id)
if not os.path.isdir(dir_path):
return
files = set()
for name in os.listdir(dir_path):
path = os.path.join(dir_path, name)
if not (os.path.isfile(path) or name == 'signatures' or locale_pattern.match(name)):
yield _("Found non-file at %s") % path
continue
files.add(name)
used = {'signatures', }
for build in app.builds:
for fname in build.patch:
if fname not in files:
yield _("Unknown file '{filename}' in build '{versionName}'")\
.format(filename=fname, versionName=build.versionName)
else:
used.add(fname)
for name in files.difference(used):
if locale_pattern.match(name):
continue
yield _("Unused file at %s") % os.path.join(dir_path, name)
def check_format(app):
if options.format and not rewritemeta.proper_format(app):
yield _("Run rewritemeta to fix formatting")
def check_license_tag(app):
'''Ensure all license tags are in https://spdx.org/license-list'''
if app.License.rstrip('+') not in SPDX:
yield _('Invalid license tag "%s"! Use only tags from https://spdx.org/license-list') \
% (app.License)
def check_extlib_dir(apps):
dir_path = os.path.join('build', 'extlib')
unused_extlib_files = set()
for root, dirs, files in os.walk(dir_path):
for name in files:
unused_extlib_files.add(os.path.join(root, name)[len(dir_path) + 1:])
used = set()
for app in apps:
for build in app.builds:
for path in build.extlibs:
if path not in unused_extlib_files:
yield _("{appid}: Unknown extlib {path} in build '{versionName}'")\
.format(appid=app.id, path=path, versionName=build.versionName)
else:
used.add(path)
for path in unused_extlib_files.difference(used):
if any(path.endswith(s) for s in [
'.gitignore',
'source.txt', 'origin.txt', 'md5.txt',
'LICENSE', 'LICENSE.txt',
'COPYING', 'COPYING.txt',
'NOTICE', 'NOTICE.txt',
]):
continue
yield _("Unused extlib at %s") % os.path.join(dir_path, path)
def check_app_field_types(app):
"""Check the fields have valid data types"""
for field in app.keys():
v = app.get(field)
t = metadata.fieldtype(field)
if v is None:
continue
elif field == 'builds':
if not isinstance(v, list):
yield(_("{appid}: {field} must be a '{type}', but it is a '{fieldtype}'!")
.format(appid=app.id, field=field,
type='list', fieldtype=v.__class__.__name__))
elif t == metadata.TYPE_LIST and not isinstance(v, list):
yield(_("{appid}: {field} must be a '{type}', but it is a '{fieldtype}!'")
.format(appid=app.id, field=field,
type='list', fieldtype=v.__class__.__name__))
elif t == metadata.TYPE_STRING and not type(v) in (str, bool, dict):
yield(_("{appid}: {field} must be a '{type}', but it is a '{fieldtype}'!")
.format(appid=app.id, field=field,
type='str', fieldtype=v.__class__.__name__))
def check_for_unsupported_metadata_files(basedir=""):
"""Checks whether any non-metadata files are in metadata/"""
global config
return_value = False
formats = config['accepted_formats']
for f in glob.glob(basedir + 'metadata/*') + glob.glob(basedir + 'metadata/.*'):
if os.path.isdir(f):
exists = False
for t in formats:
exists = exists or os.path.exists(f + '.' + t)
if not exists:
print(_('"%s/" has no matching metadata file!') % f)
return_value = True
elif os.path.splitext(f)[1][1:] in formats:
packageName = os.path.splitext(os.path.basename(f))[0]
if not common.is_valid_package_name(packageName):
print('"' + packageName + '" is an invalid package name!\n'
+ 'https://developer.android.com/studio/build/application-id')
return_value = True
else:
print('"' + f.replace(basedir, '')
+ '" is not a supported file format: (' + ','.join(formats) + ')')
return_value = True
return return_value
def check_current_version_code(app):
"""Check that the CurrentVersionCode is currently available"""
archive_policy = app.get('ArchivePolicy')
if archive_policy and archive_policy.split()[0] == "0":
return
cv = app.get('CurrentVersionCode')
if cv is not None and int(cv) == 0:
return
builds = app.get('builds')
active_builds = 0
min_versionCode = None
if builds:
for build in builds:
vc = int(build['versionCode'])
if min_versionCode is None or min_versionCode > vc:
min_versionCode = vc
if not build.get('disable'):
active_builds += 1
if cv == build['versionCode']:
break
if active_builds == 0:
return # all builds are disabled
if cv is not None and int(cv) < min_versionCode:
yield(_('CurrentVersionCode {cv} is less than oldest build entry {versionCode}')
.format(cv=cv, versionCode=min_versionCode))
def main():
global config, options
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
common.setup_global_opts(parser)
parser.add_argument("-f", "--format", action="store_true", default=False,
help=_("Also warn about formatting issues, like rewritemeta -l"))
parser.add_argument("appid", nargs='*', help=_("applicationId in the form APPID"))
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
metadata.warnings_action = options.W
config = common.read_config(options)
# Get all apps...
allapps = metadata.read_metadata(xref=True)
apps = common.read_app_args(options.appid, allapps, False)
anywarns = check_for_unsupported_metadata_files()
apps_check_funcs = []
if len(options.appid) == 0:
# otherwise it finds tons of unused extlibs
apps_check_funcs.append(check_extlib_dir)
for check_func in apps_check_funcs:
for warn in check_func(apps.values()):
anywarns = True
print(warn)
for appid, app in apps.items():
if app.Disabled:
continue
app_check_funcs = [
check_app_field_types,
check_regexes,
check_update_check_data_url,
check_vercode_operation,
check_ucm_tags,
check_char_limits,
check_old_links,
check_checkupdates_ran,
check_useless_fields,
check_empty_fields,
check_categories,
check_duplicates,
check_mediawiki_links,
check_bulleted_lists,
check_builds,
check_files_dir,
check_format,
check_license_tag,
check_current_version_code,
]
for check_func in app_check_funcs:
for warn in check_func(app):
anywarns = True
print("%s: %s" % (appid, warn))
if anywarns:
sys.exit(1)
# A compiled, public domain list of official SPDX license tags from:
# https://github.com/sindresorhus/spdx-license-list/blob/v4.0.0/spdx-simple.json
# The deprecated license tags have been removed from the list, they are at the
# bottom, starting after the last license tags that start with Z.
# This is at the bottom, since its a long list of data
SPDX = [
"PublicDomain", # an F-Droid addition, until we can enforce a better option
"0BSD",
"AAL",
"Abstyles",
"Adobe-2006",
"Adobe-Glyph",
"ADSL",
"AFL-1.1",
"AFL-1.2",
"AFL-2.0",
"AFL-2.1",
"AFL-3.0",
"Afmparse",
"AGPL-1.0",
"AGPL-3.0-only",
"AGPL-3.0-or-later",
"Aladdin",
"AMDPLPA",
"AML",
"AMPAS",
"ANTLR-PD",
"Apache-1.0",
"Apache-1.1",
"Apache-2.0",
"APAFML",
"APL-1.0",
"APSL-1.0",
"APSL-1.1",
"APSL-1.2",
"APSL-2.0",
"Artistic-1.0-cl8",
"Artistic-1.0-Perl",
"Artistic-1.0",
"Artistic-2.0",
"Bahyph",
"Barr",
"Beerware",
"BitTorrent-1.0",
"BitTorrent-1.1",
"Borceux",
"BSD-1-Clause",
"BSD-2-Clause-FreeBSD",
"BSD-2-Clause-NetBSD",
"BSD-2-Clause-Patent",
"BSD-2-Clause",
"BSD-3-Clause-Attribution",
"BSD-3-Clause-Clear",
"BSD-3-Clause-LBNL",
"BSD-3-Clause-No-Nuclear-License-2014",
"BSD-3-Clause-No-Nuclear-License",
"BSD-3-Clause-No-Nuclear-Warranty",
"BSD-3-Clause",
"BSD-4-Clause-UC",
"BSD-4-Clause",
"BSD-Protection",
"BSD-Source-Code",
"BSL-1.0",
"bzip2-1.0.5",
"bzip2-1.0.6",
"Caldera",
"CATOSL-1.1",
"CC-BY-1.0",
"CC-BY-2.0",
"CC-BY-2.5",
"CC-BY-3.0",
"CC-BY-4.0",
"CC-BY-NC-1.0",
"CC-BY-NC-2.0",
"CC-BY-NC-2.5",
"CC-BY-NC-3.0",
"CC-BY-NC-4.0",
"CC-BY-NC-ND-1.0",
"CC-BY-NC-ND-2.0",
"CC-BY-NC-ND-2.5",
"CC-BY-NC-ND-3.0",
"CC-BY-NC-ND-4.0",
"CC-BY-NC-SA-1.0",
"CC-BY-NC-SA-2.0",
"CC-BY-NC-SA-2.5",
"CC-BY-NC-SA-3.0",
"CC-BY-NC-SA-4.0",
"CC-BY-ND-1.0",
"CC-BY-ND-2.0",
"CC-BY-ND-2.5",
"CC-BY-ND-3.0",
"CC-BY-ND-4.0",
"CC-BY-SA-1.0",
"CC-BY-SA-2.0",
"CC-BY-SA-2.5",
"CC-BY-SA-3.0",
"CC-BY-SA-4.0",
"CC0-1.0",
"CDDL-1.0",
"CDDL-1.1",
"CDLA-Permissive-1.0",
"CDLA-Sharing-1.0",
"CECILL-1.0",
"CECILL-1.1",
"CECILL-2.0",
"CECILL-2.1",
"CECILL-B",
"CECILL-C",
"ClArtistic",
"CNRI-Jython",
"CNRI-Python-GPL-Compatible",
"CNRI-Python",
"Condor-1.1",
"CPAL-1.0",
"CPL-1.0",
"CPOL-1.02",
"Crossword",
"CrystalStacker",
"CUA-OPL-1.0",
"Cube",
"curl",
"D-FSL-1.0",
"diffmark",
"DOC",
"Dotseqn",
"DSDP",
"dvipdfm",
"ECL-1.0",
"ECL-2.0",
"EFL-1.0",
"EFL-2.0",
"eGenix",
"Entessa",
"EPL-1.0",
"EPL-2.0",
"ErlPL-1.1",
"EUDatagrid",
"EUPL-1.0",
"EUPL-1.1",
"EUPL-1.2",
"Eurosym",
"Fair",
"Frameworx-1.0",
"FreeImage",
"FSFAP",
"FSFUL",
"FSFULLR",
"FTL",
"GFDL-1.1-only",
"GFDL-1.1-or-later",
"GFDL-1.2-only",
"GFDL-1.2-or-later",
"GFDL-1.3-only",
"GFDL-1.3-or-later",
"Giftware",
"GL2PS",
"Glide",
"Glulxe",
"gnuplot",
"GPL-1.0-only",
"GPL-1.0-or-later",
"GPL-2.0-only",
"GPL-2.0-or-later",
"GPL-3.0-only",
"GPL-3.0-or-later",
"gSOAP-1.3b",
"HaskellReport",
"HPND",
"IBM-pibs",
"ICU",
"IJG",
"ImageMagick",
"iMatix",
"Imlib2",
"Info-ZIP",
"Intel-ACPI",
"Intel",
"Interbase-1.0",
"IPA",
"IPL-1.0",
"ISC",
"JasPer-2.0",
"JSON",
"LAL-1.2",
"LAL-1.3",
"Latex2e",
"Leptonica",
"LGPL-2.0-only",
"LGPL-2.0-or-later",
"LGPL-2.1-only",
"LGPL-2.1-or-later",
"LGPL-3.0-only",
"LGPL-3.0-or-later",
"LGPLLR",
"Libpng",
"libtiff",
"LiLiQ-P-1.1",
"LiLiQ-R-1.1",
"LiLiQ-Rplus-1.1",
"LPL-1.0",
"LPL-1.02",
"LPPL-1.0",
"LPPL-1.1",
"LPPL-1.2",
"LPPL-1.3a",
"LPPL-1.3c",
"MakeIndex",
"MirOS",
"MIT-advertising",
"MIT-CMU",
"MIT-enna",
"MIT-feh",
"MIT",
"MITNFA",
"Motosoto",
"mpich2",
"MPL-1.0",
"MPL-1.1",
"MPL-2.0-no-copyleft-exception",
"MPL-2.0",
"MS-PL",
"MS-RL",
"MTLL",
"Multics",
"Mup",
"NASA-1.3",
"Naumen",
"NBPL-1.0",
"NCSA",
"Net-SNMP",
"NetCDF",
"Newsletr",
"NGPL",
"NLOD-1.0",
"NLPL",
"Nokia",
"NOSL",
"Noweb",
"NPL-1.0",
"NPL-1.1",
"NPOSL-3.0",
"NRL",
"NTP",
"OCCT-PL",
"OCLC-2.0",
"ODbL-1.0",
"OFL-1.0",
"OFL-1.1",
"OGTSL",
"OLDAP-1.1",
"OLDAP-1.2",
"OLDAP-1.3",
"OLDAP-1.4",
"OLDAP-2.0.1",
"OLDAP-2.0",
"OLDAP-2.1",
"OLDAP-2.2.1",
"OLDAP-2.2.2",
"OLDAP-2.2",
"OLDAP-2.3",
"OLDAP-2.4",
"OLDAP-2.5",
"OLDAP-2.6",
"OLDAP-2.7",
"OLDAP-2.8",
"OML",
"OpenSSL",
"OPL-1.0",
"OSET-PL-2.1",
"OSL-1.0",
"OSL-1.1",
"OSL-2.0",
"OSL-2.1",
"OSL-3.0",
"PDDL-1.0",
"PHP-3.0",
"PHP-3.01",
"Plexus",
"PostgreSQL",
"psfrag",
"psutils",
"Python-2.0",
"Qhull",
"QPL-1.0",
"Rdisc",
"RHeCos-1.1",
"RPL-1.1",
"RPL-1.5",
"RPSL-1.0",
"RSA-MD",
"RSCPL",
"Ruby",
"SAX-PD",
"Saxpath",
"SCEA",
"Sendmail",
"SGI-B-1.0",
"SGI-B-1.1",
"SGI-B-2.0",
"SimPL-2.0",
"SISSL-1.2",
"SISSL",
"Sleepycat",
"SMLNJ",
"SMPPL",
"SNIA",
"Spencer-86",
"Spencer-94",
"Spencer-99",
"SPL-1.0",
"SugarCRM-1.1.3",
"SWL",
"TCL",
"TCP-wrappers",
"TMate",
"TORQUE-1.1",
"TOSL",
"Unicode-DFS-2015",
"Unicode-DFS-2016",
"Unicode-TOU",
"Unlicense",
"UPL-1.0",
"Vim",
"VOSTROM",
"VSL-1.0",
"W3C-19980720",
"W3C-20150513",
"W3C",
"Watcom-1.0",
"Wsuipa",
"WTFPL",
"X11",
"Xerox",
"XFree86-1.1",
"xinetd",
"Xnet",
"xpp",
"XSkat",
"YPL-1.0",
"YPL-1.1",
"Zed",
"Zend-2.0",
"Zimbra-1.3",
"Zimbra-1.4",
"zlib-acknowledgement",
"Zlib",
"ZPL-1.1",
"ZPL-2.0",
"ZPL-2.1",
]
if __name__ == "__main__":
main()
fdroidserver-1.1.6/fdroidserver/metadata.py 0000644 0001750 0001750 00000144670 13576156546 021007 0 ustar hans hans 0000000 0000000 #!/usr/bin/env python3
#
# metadata.py - part of the FDroid server tools
# Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013-2014 Daniel Martí
# Copyright (C) 2017-2018 Michael Pöhn
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
import json
import os
import re
import glob
import html
import logging
import textwrap
import io
import yaml
from collections import OrderedDict
import fdroidserver.common
from fdroidserver import _
from fdroidserver.exception import MetaDataException, FDroidException
srclibs = None
warnings_action = None
def warn_or_exception(value):
'''output warning or Exception depending on -W'''
if warnings_action == 'ignore':
pass
elif warnings_action == 'error':
raise MetaDataException(value)
else:
logging.warning(value)
# To filter which ones should be written to the metadata files if
# present
app_fields = set([
'Disabled',
'AntiFeatures',
'Provides',
'Categories',
'License',
'Author Name',
'Author Email',
'Author Web Site',
'Web Site',
'Source Code',
'Issue Tracker',
'Translation',
'Changelog',
'Donate',
'FlattrID',
'LiberapayID',
'Bitcoin',
'Litecoin',
'Name',
'Auto Name',
'Summary',
'Description',
'Requires Root',
'Repo Type',
'Repo',
'Binaries',
'Maintainer Notes',
'Archive Policy',
'Auto Update Mode',
'Update Check Mode',
'Update Check Ignore',
'Vercode Operation',
'Update Check Name',
'Update Check Data',
'Current Version',
'Current Version Code',
'No Source Since',
'Build',
'comments', # For formats that don't do inline comments
'builds', # For formats that do builds as a list
])
yaml_app_field_order = [
'Disabled',
'AntiFeatures',
'Provides',
'Categories',
'License',
'AuthorName',
'AuthorEmail',
'AuthorWebSite',
'WebSite',
'SourceCode',
'IssueTracker',
'Translation',
'Changelog',
'Donate',
'FlattrID',
'LiberapayID',
'Bitcoin',
'Litecoin',
'\n',
'Name',
'AutoName',
'Summary',
'Description',
'\n',
'RequiresRoot',
'\n',
'RepoType',
'Repo',
'Binaries',
'\n',
'Builds',
'\n',
'MaintainerNotes',
'\n',
'ArchivePolicy',
'AutoUpdateMode',
'UpdateCheckMode',
'UpdateCheckIgnore',
'VercodeOperation',
'UpdateCheckName',
'UpdateCheckData',
'CurrentVersion',
'CurrentVersionCode',
'\n',
'NoSourceSince',
]
yaml_app_fields = [x for x in yaml_app_field_order if x != '\n']
class App(dict):
def __init__(self, copydict=None):
if copydict:
super().__init__(copydict)
return
super().__init__()
self.Disabled = None
self.AntiFeatures = []
self.Provides = None
self.Categories = []
self.License = 'Unknown'
self.AuthorName = None
self.AuthorEmail = None
self.AuthorWebSite = None
self.WebSite = ''
self.SourceCode = ''
self.IssueTracker = ''
self.Translation = ''
self.Changelog = ''
self.Donate = None
self.FlattrID = None
self.LiberapayID = None
self.Bitcoin = None
self.Litecoin = None
self.Name = None
self.AutoName = ''
self.Summary = ''
self.Description = ''
self.RequiresRoot = False
self.RepoType = ''
self.Repo = ''
self.Binaries = None
self.MaintainerNotes = ''
self.ArchivePolicy = None
self.AutoUpdateMode = 'None'
self.UpdateCheckMode = 'None'
self.UpdateCheckIgnore = None
self.VercodeOperation = None
self.UpdateCheckName = None
self.UpdateCheckData = None
self.CurrentVersion = ''
self.CurrentVersionCode = None
self.NoSourceSince = ''
self.id = None
self.metadatapath = None
self.builds = []
self.comments = {}
self.added = None
self.lastUpdated = None
def __getattr__(self, name):
if name in self:
return self[name]
else:
raise AttributeError("No such attribute: " + name)
def __setattr__(self, name, value):
self[name] = value
def __delattr__(self, name):
if name in self:
del self[name]
else:
raise AttributeError("No such attribute: " + name)
def get_last_build(self):
if len(self.builds) > 0:
return self.builds[-1]
else:
return Build()
TYPE_UNKNOWN = 0
TYPE_OBSOLETE = 1
TYPE_STRING = 2
TYPE_BOOL = 3
TYPE_LIST = 4
TYPE_SCRIPT = 5
TYPE_MULTILINE = 6
TYPE_BUILD = 7
TYPE_INT = 8
fieldtypes = {
'Description': TYPE_MULTILINE,
'MaintainerNotes': TYPE_MULTILINE,
'Categories': TYPE_LIST,
'AntiFeatures': TYPE_LIST,
'Build': TYPE_BUILD,
'BuildVersion': TYPE_OBSOLETE,
'UseBuilt': TYPE_OBSOLETE,
}
def fieldtype(name):
name = name.replace(' ', '')
if name in fieldtypes:
return fieldtypes[name]
return TYPE_STRING
# In the order in which they are laid out on files
build_flags_order = [
'disable',
'commit',
'timeout',
'subdir',
'submodules',
'sudo',
'init',
'patch',
'gradle',
'maven',
'buildozer',
'output',
'srclibs',
'oldsdkloc',
'encoding',
'forceversion',
'forcevercode',
'rm',
'extlibs',
'prebuild',
'androidupdate',
'target',
'scanignore',
'scandelete',
'build',
'buildjni',
'ndk',
'preassemble',
'gradleprops',
'antcommands',
'novcheck',
'antifeatures',
]
# old .txt format has version name/code inline in the 'Build:' line
# but YAML and JSON have a explicit key for them
build_flags = ['versionName', 'versionCode'] + build_flags_order
class Build(dict):
def __init__(self, copydict=None):
super().__init__()
self.disable = ''
self.commit = None
self.timeout = None
self.subdir = None
self.submodules = False
self.sudo = ''
self.init = ''
self.patch = []
self.gradle = []
self.maven = False
self.buildozer = False
self.output = None
self.srclibs = []
self.oldsdkloc = False
self.encoding = None
self.forceversion = False
self.forcevercode = False
self.rm = []
self.extlibs = []
self.prebuild = ''
self.androidupdate = []
self.target = None
self.scanignore = []
self.scandelete = []
self.build = ''
self.buildjni = []
self.ndk = None
self.preassemble = []
self.gradleprops = []
self.antcommands = []
self.novcheck = False
self.antifeatures = []
if copydict:
super().__init__(copydict)
return
def __getattr__(self, name):
if name in self:
return self[name]
else:
raise AttributeError("No such attribute: " + name)
def __setattr__(self, name, value):
self[name] = value
def __delattr__(self, name):
if name in self:
del self[name]
else:
raise AttributeError("No such attribute: " + name)
def build_method(self):
for f in ['maven', 'gradle', 'buildozer']:
if self.get(f):
return f
if self.output:
return 'raw'
return 'ant'
# like build_method, but prioritize output=
def output_method(self):
if self.output:
return 'raw'
for f in ['maven', 'gradle', 'buildozer']:
if self.get(f):
return f
return 'ant'
def ndk_path(self):
version = self.ndk
if not version:
version = 'r12b' # falls back to latest
paths = fdroidserver.common.config['ndk_paths']
if version not in paths:
return ''
return paths[version]
flagtypes = {
'versionCode': TYPE_INT,
'extlibs': TYPE_LIST,
'srclibs': TYPE_LIST,
'patch': TYPE_LIST,
'rm': TYPE_LIST,
'buildjni': TYPE_LIST,
'preassemble': TYPE_LIST,
'androidupdate': TYPE_LIST,
'scanignore': TYPE_LIST,
'scandelete': TYPE_LIST,
'gradle': TYPE_LIST,
'antcommands': TYPE_LIST,
'gradleprops': TYPE_LIST,
'sudo': TYPE_SCRIPT,
'init': TYPE_SCRIPT,
'prebuild': TYPE_SCRIPT,
'build': TYPE_SCRIPT,
'submodules': TYPE_BOOL,
'oldsdkloc': TYPE_BOOL,
'forceversion': TYPE_BOOL,
'forcevercode': TYPE_BOOL,
'novcheck': TYPE_BOOL,
'antifeatures': TYPE_LIST,
'timeout': TYPE_INT,
}
def flagtype(name):
if name in flagtypes:
return flagtypes[name]
return TYPE_STRING
class FieldValidator():
"""
Designates App metadata field types and checks that it matches
'name' - The long name of the field type
'matching' - List of possible values or regex expression
'sep' - Separator to use if value may be a list
'fields' - Metadata fields (Field:Value) of this type
"""
def __init__(self, name, matching, fields):
self.name = name
self.matching = matching
self.compiled = re.compile(matching)
self.fields = fields
def check(self, v, appid):
if not v:
return
if type(v) == list:
values = v
else:
values = [v]
for v in values:
if not self.compiled.match(v):
warn_or_exception(_("'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}")
.format(value=v, field=self.name, appid=appid, pattern=self.matching))
# Generic value types
valuetypes = {
FieldValidator("Flattr ID",
r'^[0-9a-z]+$',
['FlattrID']),
FieldValidator("Liberapay ID",
r'^[0-9]+$',
['LiberapayID']),
FieldValidator("HTTP link",
r'^http[s]?://',
["WebSite", "SourceCode", "IssueTracker", "Translation", "Changelog", "Donate"]),
FieldValidator("Email",
r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
["AuthorEmail"]),
FieldValidator("Bitcoin address",
r'^(bc1|[13])[a-zA-HJ-NP-Z0-9]{25,39}$',
["Bitcoin"]),
FieldValidator("Litecoin address",
r'^L[a-zA-Z0-9]{33}$',
["Litecoin"]),
FieldValidator("Repo Type",
r'^(git|git-svn|svn|hg|bzr|srclib)$',
["RepoType"]),
FieldValidator("Binaries",
r'^http[s]?://',
["Binaries"]),
FieldValidator("Archive Policy",
r'^[0-9]+ versions$',
["ArchivePolicy"]),
FieldValidator("Anti-Feature",
r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln|ApplicationDebuggable|NoSourceSince)$',
["AntiFeatures"]),
FieldValidator("Auto Update Mode",
r"^(Version .+|None)$",
["AutoUpdateMode"]),
FieldValidator("Update Check Mode",
r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$",
["UpdateCheckMode"])
}
# Check an app's metadata information for integrity errors
def check_metadata(app):
for v in valuetypes:
for k in v.fields:
v.check(app[k], app.id)
# Formatter for descriptions. Create an instance, and call parseline() with
# each line of the description source from the metadata. At the end, call
# end() and then text_txt and text_html will contain the result.
class DescriptionFormatter:
stNONE = 0
stPARA = 1
stUL = 2
stOL = 3
def __init__(self, linkres):
self.bold = False
self.ital = False
self.state = self.stNONE
self.laststate = self.stNONE
self.text_html = ''
self.text_txt = ''
self.html = io.StringIO()
self.text = io.StringIO()
self.para_lines = []
self.linkResolver = None
self.linkResolver = linkres
def endcur(self, notstates=None):
if notstates and self.state in notstates:
return
if self.state == self.stPARA:
self.endpara()
elif self.state == self.stUL:
self.endul()
elif self.state == self.stOL:
self.endol()
def endpara(self):
self.laststate = self.state
self.state = self.stNONE
whole_para = ' '.join(self.para_lines)
self.addtext(whole_para)
wrapped = textwrap.fill(whole_para, 80,
break_long_words=False,
break_on_hyphens=False)
self.text.write(wrapped)
self.html.write('')
del self.para_lines[:]
def endul(self):
self.html.write('')
self.laststate = self.state
self.state = self.stNONE
def endol(self):
self.html.write('')
self.laststate = self.state
self.state = self.stNONE
def formatted(self, txt, htmlbody):
res = ''
if htmlbody:
txt = html.escape(txt, quote=False)
while True:
index = txt.find("''")
if index == -1:
return res + txt
res += txt[:index]
txt = txt[index:]
if txt.startswith("'''"):
if htmlbody:
if self.bold:
res += ''
else:
res += ''
self.bold = not self.bold
txt = txt[3:]
else:
if htmlbody:
if self.ital:
res += ''
else:
res += ''
self.ital = not self.ital
txt = txt[2:]
def linkify(self, txt):
res_plain = ''
res_html = ''
while True:
index = txt.find("[")
if index == -1:
return (res_plain + self.formatted(txt, False), res_html + self.formatted(txt, True))
res_plain += self.formatted(txt[:index], False)
res_html += self.formatted(txt[:index], True)
txt = txt[index:]
if txt.startswith("[["):
index = txt.find("]]")
if index == -1:
warn_or_exception(_("Unterminated ]]"))
url = txt[2:index]
if self.linkResolver:
url, urltext = self.linkResolver(url)
else:
urltext = url
res_html += '' + html.escape(urltext, quote=False) + ''
res_plain += urltext
txt = txt[index + 2:]
else:
index = txt.find("]")
if index == -1:
warn_or_exception(_("Unterminated ]"))
url = txt[1:index]
index2 = url.find(' ')
if index2 == -1:
urltxt = url
else:
urltxt = url[index2 + 1:]
url = url[:index2]
if url == urltxt:
warn_or_exception(_("URL title is just the URL, use brackets: [URL]"))
res_html += '' + html.escape(urltxt, quote=False) + ''
res_plain += urltxt
if urltxt != url:
res_plain += ' (' + url + ')'
txt = txt[index + 1:]
def addtext(self, txt):
p, h = self.linkify(txt)
self.html.write(h)
def parseline(self, line):
if not line:
self.endcur()
elif line.startswith('* '):
self.endcur([self.stUL])
if self.state != self.stUL:
self.html.write('