pax_global_header 0000666 0000000 0000000 00000000064 14160667064 0014524 g ustar 00root root 0000000 0000000 52 comment=dcf0825338889c5599932fb89a4aed8fba9b770b
appstream-generator-0.8.6/ 0000775 0000000 0000000 00000000000 14160667064 0015517 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/.editorconfig 0000664 0000000 0000000 00000000363 14160667064 0020176 0 ustar 00root root 0000000 0000000 # See https://editorconfig.org/
root = true
[*]
end_of_line = lf
trim_trailing_whitespace = true
charset = utf-8
indent_style = space
indent_size = 4
[*.yml]
indent_style = space
indent_size = 2
[*.xml]
indent_style = space
indent_size = 2
appstream-generator-0.8.6/.github/ 0000775 0000000 0000000 00000000000 14160667064 0017057 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/.github/workflows/ 0000775 0000000 0000000 00000000000 14160667064 0021114 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/.github/workflows/build-test.yml 0000664 0000000 0000000 00000002567 14160667064 0023725 0 ustar 00root root 0000000 0000000 name: Build Test
on: [push, pull_request]
jobs:
build-debian-testing:
name: Debian Testing
runs-on: ubuntu-latest
strategy:
matrix:
dc: [ldc, gdc]
steps:
- uses: actions/checkout@v2
- name: Create Build Environment
run: cd tests/ci/ && podman build -t asgen -f ./Dockerfile-debian-testing .
- name: Build & Test
continue-on-error: ${{ matrix.dc == 'gdc' }}
run: podman run -a stdout -a stderr -e DC=${{ matrix.dc }} -v `pwd`:/build asgen
./tests/ci/build-and-test.sh
build-fedora-rawhide:
name: Fedora Rawhide
runs-on: ubuntu-latest
strategy:
matrix:
dc: [ldc, gdc]
steps:
- uses: actions/checkout@v2
- name: Create Build Environment
run: cd tests/ci/ && podman build -t asgen -f ./Dockerfile-fedora-rawhide .
- name: Build & Test
continue-on-error: ${{ matrix.dc == 'gdc' }}
run: podman run -a stdout -a stderr -e DC=${{ matrix.dc }} -v `pwd`:/build asgen
./tests/ci/build-and-test.sh
build-ubuntu-lts:
name: Ubuntu LTS
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- name: Create Build Environment
run: sudo ./tests/ci/install-deps-deb.sh
- name: Make & Install 3rd-party
run: sudo ./tests/ci/ci-install-extern.sh
- name: Build & Test
run: DC=ldc2 ./tests/ci/build-and-test.sh
appstream-generator-0.8.6/.gitignore 0000664 0000000 0000000 00000000112 14160667064 0017501 0 ustar 00root root 0000000 0000000 build/
.dub
docs.json
dub.selections.json
__dummy.html
*.o
*.obj
*.so
*.a
appstream-generator-0.8.6/LICENSE 0000664 0000000 0000000 00000016743 14160667064 0016537 0 ustar 00root root 0000000 0000000 GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc.
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
This version of the GNU Lesser General Public License incorporates
the terms and conditions of version 3 of the GNU General Public
License, supplemented by the additional permissions listed below.
0. Additional Definitions.
As used herein, "this License" refers to version 3 of the GNU Lesser
General Public License, and the "GNU GPL" refers to version 3 of the GNU
General Public License.
"The Library" refers to a covered work governed by this License,
other than an Application or a Combined Work as defined below.
An "Application" is any work that makes use of an interface provided
by the Library, but which is not otherwise based on the Library.
Defining a subclass of a class defined by the Library is deemed a mode
of using an interface provided by the Library.
A "Combined Work" is a work produced by combining or linking an
Application with the Library. The particular version of the Library
with which the Combined Work was made is also called the "Linked
Version".
The "Minimal Corresponding Source" for a Combined Work means the
Corresponding Source for the Combined Work, excluding any source code
for portions of the Combined Work that, considered in isolation, are
based on the Application, and not on the Linked Version.
The "Corresponding Application Code" for a Combined Work means the
object code and/or source code for the Application, including any data
and utility programs needed for reproducing the Combined Work from the
Application, but excluding the System Libraries of the Combined Work.
1. Exception to Section 3 of the GNU GPL.
You may convey a covered work under sections 3 and 4 of this License
without being bound by section 3 of the GNU GPL.
2. Conveying Modified Versions.
If you modify a copy of the Library, and, in your modifications, a
facility refers to a function or data to be supplied by an Application
that uses the facility (other than as an argument passed when the
facility is invoked), then you may convey a copy of the modified
version:
a) under this License, provided that you make a good faith effort to
ensure that, in the event an Application does not supply the
function or data, the facility still operates, and performs
whatever part of its purpose remains meaningful, or
b) under the GNU GPL, with none of the additional permissions of
this License applicable to that copy.
3. Object Code Incorporating Material from Library Header Files.
The object code form of an Application may incorporate material from
a header file that is part of the Library. You may convey such object
code under terms of your choice, provided that, if the incorporated
material is not limited to numerical parameters, data structure
layouts and accessors, or small macros, inline functions and templates
(ten or fewer lines in length), you do both of the following:
a) Give prominent notice with each copy of the object code that the
Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the object code with a copy of the GNU GPL and this license
document.
4. Combined Works.
You may convey a Combined Work under terms of your choice that,
taken together, effectively do not restrict modification of the
portions of the Library contained in the Combined Work and reverse
engineering for debugging such modifications, if you also do each of
the following:
a) Give prominent notice with each copy of the Combined Work that
the Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the Combined Work with a copy of the GNU GPL and this license
document.
c) For a Combined Work that displays copyright notices during
execution, include the copyright notice for the Library among
these notices, as well as a reference directing the user to the
copies of the GNU GPL and this license document.
d) Do one of the following:
0) Convey the Minimal Corresponding Source under the terms of this
License, and the Corresponding Application Code in a form
suitable for, and under terms that permit, the user to
recombine or relink the Application with a modified version of
the Linked Version to produce a modified Combined Work, in the
manner specified by section 6 of the GNU GPL for conveying
Corresponding Source.
1) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (a) uses at run time
a copy of the Library already present on the user's computer
system, and (b) will operate properly with a modified version
of the Library that is interface-compatible with the Linked
Version.
e) Provide Installation Information, but only if you would otherwise
be required to provide such information under section 6 of the
GNU GPL, and only to the extent that such information is
necessary to install and execute a modified version of the
Combined Work produced by recombining or relinking the
Application with a modified version of the Linked Version. (If
you use option 4d0, the Installation Information must accompany
the Minimal Corresponding Source and Corresponding Application
Code. If you use option 4d1, you must provide the Installation
Information in the manner specified by section 6 of the GNU GPL
for conveying Corresponding Source.)
5. Combined Libraries.
You may place library facilities that are a work based on the
Library side by side in a single library together with other library
facilities that are not Applications and are not covered by this
License, and convey such a combined library under terms of your
choice, if you do both of the following:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities,
conveyed under the terms of this License.
b) Give prominent notice with the combined library that part of it
is a work based on the Library, and explaining where to find the
accompanying uncombined form of the same work.
6. Revised Versions of the GNU Lesser General Public License.
The Free Software Foundation may publish revised and/or new versions
of the GNU Lesser General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the
Library as you received it specifies that a certain numbered version
of the GNU Lesser General Public License "or any later version"
applies to it, you have the option of following the terms and
conditions either of that published version or of any later version
published by the Free Software Foundation. If the Library as you
received it does not specify a version number of the GNU Lesser
General Public License, you may choose any version of the GNU Lesser
General Public License ever published by the Free Software Foundation.
If the Library as you received it specifies that a proxy can decide
whether future versions of the GNU Lesser General Public License shall
apply, that proxy's public statement of acceptance of any version is
permanent authorization for you to choose that version for the
Library.
appstream-generator-0.8.6/MAINTAINERS 0000664 0000000 0000000 00000000047 14160667064 0017215 0 ustar 00root root 0000000 0000000 Matthias Klumpp
E-mail: mak@debian.org
appstream-generator-0.8.6/NEWS 0000664 0000000 0000000 00000066755 14160667064 0016241 0 ustar 00root root 0000000 0000000 Version 0.8.6
~~~~~~~~~~~~~~
Released: 2021-12-22
Notes:
* This is the first release to fully use libappstream-compose for
almost all metadata analysis (except icon processing, which needs
special care).
Features:
* Reimplement data extractor using the compose API
* Enable screencasts in compose based on our screenshotVideos feature flag
* Run external metadata through the exact same processing as all other data
* Redirect glib debug messages to our own logging if in verbose mode
* Adjust parallel processing work unit sizes for seeding
* ci: Add Fedora to the CI environment
Bugfixes:
* Fix build with GLib >= 2.69
* engine: Protect iconTarFiles variable when scanning for icons in parallel
* Resolve some issue tag conflicts with as-compose
* Control batch processing better to enable better extractor reuse
* Add duplicate-metainfo exemption for injected data
* Improve explanation of the "no-metainfo" tag
* Don't emit "description-from-package" if "no-metainfo" was already present
Contributors:
Neal Gompa (ニール・ゴンパ), Matthias Klumpp
Version 0.8.5
~~~~~~~~~~~~~~
Released: 2021-08-28
Features:
* snap: Set `grade` to `stable`
* Improve metainfo/desktop deduplication code
* Use ascompose for adding translation status data
Bugfixes:
* Always mark desktop-file as handled when handling its metainfo file (#91)
* Fix compilation due to broken generated GdkPixbuf code
* config: Move setUseOptipng call after we determine if optipng is available
* snap: Add glib{,-networking} to asgen's stage-packages
* snap: Build libas from a tag
* Adjust for compose API changes
Contributors:
Balló György, Dan Printzell, Iain Lane and Matthias Klumpp
Version 0.8.4
~~~~~~~~~~~~~~
Released: 2021-03-02
Features:
* Replace our HashMap with native associative arrays
* Switch back to using the curl-based downloader again
* Use the desktop-entry parsing code from appstream-compose
* Try finding icons in /usr/share/icons root directory as well
* Update our own metainfo file and validate it
Bugfixes:
* snap: Set LD_LIBRARY_PATH to the stage dir for asgen's build
* snap: Install curl as a normal build-package
* snap: Build with -j1
* debian: Forget a package's filename when removing its temporary dir
* debian: Add more synchronization
* zarchive: Log the error when we can't open an archive
* various: Improve builddir != srcdir builds and build the snap like this
* Replace theme icon generator concurrency with a range
* snap: Build AppStream in debugoptimized mode instead of release mode
Contributors:
Iain Lane, Matthias Klumpp
Version 0.8.3
~~~~~~~~~~~~~~
Released: 2021-02-02
Notes:
* This release requires libappstream-compose from the AppStream project to build.
The library is still in progress and currently has an unstable API, but
is developed in sync with appstream-generator, so asgen can safely
depend on it.
Features:
* Elevate no-metainfo presence to a warning by default (Matthias Klumpp)
* Ignore all apps with only desktop-entry files using OnlyShowIn (Matthias Klumpp)
* Make use of the helper classes provided by appstream-compose (Matthias Klumpp)
* Add editorconfig (Matthias Klumpp)
* Use ascompose result type as base for the generator result container (Matthias Klumpp)
* Use the validator helper function from appstream-compose (Matthias Klumpp)
* Use metainfo parsing helper from appstream-compose (Matthias Klumpp)
* alpine: add capabilities to download packages via HTTP (Rasmus Thomsen)
* config: allow overriding export dir via --export-dir (Rasmus Thomsen)
Bugfixes:
* Add explicit option to disable network-dependent tests (Matthias Klumpp)
* Add missing CSS styling for permalinks to Debian template (Matthias Klumpp)
* Captialize "MetaInfo" the same everywhere in hint messages (Matthias Klumpp)
* Use binding generator to create the missing AppStream Utils functions (Matthias Klumpp)
* Never open contents cache DB env more than once in the same thread (Matthias Klumpp)
* Re-enable LMDB TLS support (Matthias Klumpp)
* downloader: Check read byte count before appending to buffer (Matthias Klumpp)
* Ensure export directory paths are sane, absolute paths all the time (Matthias Klumpp)
Version 0.8.2
~~~~~~~~~~~~~~
Released: 2020-05-12
Features:
* Add experimental Snapcraft build definition
* snap: Move to strict confinement (Iain Lane)
* snap: Lots of debugging & integration work (Iain Lane)
* Find external binaries in PATH instead of hardcoding absolute paths to them
* Add static bindings for libsoup
* Switch to new libsoup-based downloader, drop Curl
* Always log the generator version for some operations
* Display more verbose debug messages when downloads are retried
* Better messages for network connection errors
* Support Ubuntu language packs for l10n status info extraction
* Make permalink anchors visible in HTML output
* alpine: Add new backend for Alpine Linux (Rasmus Thomsen)
Bugfixes:
* Pull a few Mustache engine code improvements from upstream
* Throw a better error message in unit tests when ffprobe isn't found
* tests: Skip video metadata check if ffprobe can not be found
* Check if our GdkPixbuf is able to handle all essential image formats
* Slighty reduce default logging verbosity when finding icons
* Give up icon search when component was rejected in icon storage routine
* Swallow bad last-modified times from servers instead of freaking out
* Make downloader a thread-local singleton (instead of thread-global)
* Create XDG runtime dir in case it is missing
* Make a failure to read an archive symlink target non-fatal
* Print available font names in issue report if no matching font was found
* Handle unexpected NULL-byte files better
* Find Yarn, even when it was renamed
* Strip out release artifacts for components that have a package
Version 0.8.1
~~~~~~~~~~~~~~
Released: 2020-01-20
Bugfixes:
* Don't crash if a gettext locale has no strings
* Reject packages which should have a long description but don't have one
* debian: Search for bz2 files again, i18n still uses those
* Work around some curl/D behavior changes and make HTTPS downloads work again
* Check suite status before removing hints/components
* Don't fail download on HTTP status code 302
* Properly fail icon search in all failure cases, add more debug logging
* Ensure icon tarballs are closed immediately after writing to them
* Flip around sorting of suites on HTML reports
Version 0.8.0
~~~~~~~~~~~~~~
Released: 2019-09-24
Notes:
* This release needs FFmpeg installed if video screenshots should
be permitted. No transcoding will be done. Check out the options
to modify this feature in the documentation.
Features:
* Add Igor Khasilev's container implementations to the source
* Replace EMSI containers with built-in implementation
* Bump dependency on AppStream
* Fix build with latest AppStream
* Implement improved validation issue reporting
* Always build registry of all hint tags, instead of adding them on-demand
* Add AV1 video sample for video support tests
* Add video tag support, analyze and store videos
* Use generic download method for downloading screenshots as well
Bugfixes:
* Make dscanner happy again
* Drop wrap files: We don't need them anymore
* debian: Don't check for bz2 compressed indices
* Only add remote icons if we have a mediaBaseUrl and are permitting image caches
* Validate injected metainfo files as well
* Ignore pedantic issues in HTML reports
* Fix getFileContents when acting on remote data
* XML-escape validator explanations
Version 0.7.7
~~~~~~~~~~~~~~
Released: 2019-02-24
Features:
* Speed up locale search by moving it to its own database cache (Matthias Klumpp)
Bugfixes:
* Component removal requests don't need to have a package-name set (Matthias Klumpp)
Version 0.7.6
~~~~~~~~~~~~~~
Released: 2019-01-10
Features:
* Limit the amount of release metadata in output (Matthias Klumpp)
* Implement icon loading for injected metadata (Matthias Klumpp)
* Prefer some SFNT font metadata over internal heuristics (Matthias Klumpp)
* Implement Gettext locale processing (Matthias Klumpp)
* Use a struct instead of bitfield to store enabled generator features (Matthias Klumpp)
* Refactor locale statistics loader (Matthias Klumpp)
* Explicitly remove temporary package data after reading locale info (Matthias Klumpp)
* Make arch-specific injected metadata override arch:all data (Matthias Klumpp)
Bugfixes:
* Complain if the user wants to associate a component with a fake package (Matthias Klumpp)
* Check that fonts and OS components have an icon (Matthias Klumpp)
* Sanitize image URLs before attempting download (Matthias Klumpp)
Version 0.7.5
~~~~~~~~~~~~~~
Released: 2019-01-04
Features:
* Load components for Arch Linux (Balló György)
* Embed mustache-d code (Matthias Klumpp)
* Protect against HTTPS-to-HTTP downgrades (Matthias Klumpp)
* Allow asgen maintainer to mark components for removal and inject
additional metainfo files (Matthias Klumpp)
* docs: Document the metainfo injection feature (Matthias Klumpp)
Bugfixes:
* Fix a few issues found by dscanner (Matthias Klumpp)
* Properly error out on HTTP 404 status codes (Matthias Klumpp)
* Add wrapfile for mir-core (Matthias Klumpp)
* Also ignore web applications without an icon (Matthias Klumpp)
Version 0.7.4
~~~~~~~~~~~~~~
Released: 2018-08-04
Features:
* Write CID<->GCID mapping table as additional output data (Matthias Klumpp)
* Don't rebuild GLibD as part of asgen, use the shared library instead (Matthias Klumpp)
* Add hint to add launchable tag if .desktop file is missing (Matthias Klumpp)
* Use posix_spawn codepath for optipng if possible via GLib (Matthias Klumpp)
* Unconditionally add stock-type icon if desktop-file allows for it (Matthias Klumpp)
* Allow font languages to be specified in a languages tag (Matthias Klumpp)
* Improve font language processing with metainfo hints (Matthias Klumpp)
* fonts: Sort selected font styles, prefer regular style for samples (Matthias Klumpp)
* fonts: Use a random pangram for fonts supporting English (Matthias Klumpp)
* fonts: Assume 100% language support for all locale mentioned in font data (Matthias Klumpp)
Bugfixes:
* Fix build with phobos 2.081 (Antonio Rojas)
* Depend on non-broken Meson version (Matthias Klumpp)
* fonts: Always prefer English for samples if font supports it (Matthias Klumpp)
* Initialize font icon lookup table only if there are fonts (Matthias Klumpp)
Version 0.7.3
~~~~~~~~~~~~~~
Released: 2018-04-26
Features:
* Don't include desktop files with an empty OnlyShowIn and complain about that
* Improve decompression code slightly
* Improve code to check for inclusion of .desktop files
* Optimize category filter function
* ubuntu: Use GC managed arrays for langpack info storage
* ubuntu: Hold copy of language pack array per package
* ubuntu: Don't load language packs multiple times, cleanup index data properly
* Close package from a base suite immediately in the seeding step
Bugfixes:
* debian: Make package index threadsafe again
* Don't add data to contents store in parallel
* Don't add the same architecture multiple times when generating reports
* Enable static check for poor exception handling
* Don't run through a synchronized section when fetching package filename
* Update dscanner and invoke it from a more versatile Python script
Version 0.7.2
~~~~~~~~~~~~~~
Released: 2018-04-16
Features:
* Dramatically reduce memory usage of cruft cleanup operation
* Parallelize database cleanup a bit more
* debian: Add basic in-memory deduplication of translated texts for packages
Bugfixes:
* Fix build on 32bit architectures
* Make GStreamer entry entirely optional for packages
* Ensure we properly merge in desktop-entry data from launchable tags
* Never process desktop files twice
* Don't append desktop data to mime/category lists
* Update wrap files to work with gir-to-d >= 0.15
Version 0.7.1
~~~~~~~~~~~~~~
Released: 2018-04-06
Features:
* debian: Use TagFile class from Laniakea
* debian: Use the packages real architecture in its identifier
* debian: Implement packageForFile backend feature
* Do more complex parsing of metainfo license expressions
* ci: Enable more static analysis checks
* Improve manual page
Bugfixes:
* Never upscale icons if we don't need to, use pixmaps directory as last resort
* Update wrap files to work with gir-to-d >= 0.14
* Don't try to upscale tiny icons from the pixmaps directory
* Register HiDPI icons for fonts properly
* Fix some minor style issues found by D-Scanner
* Document WorkspaceDir config file option
Version 0.7.0
~~~~~~~~~~~~~~
Released: 2018-04-04
Notes:
* Processing single packages and calling publication and processing
separately is not a fully supported feature yet!
This release lays the groundwork for it, but its final implementation
will happen later. Do not use this in production!
Features:
* stdx.allocator-backed containers library
* Use stdx.allocator-backed HashMap in more places
* Trigger GC less aggressively, use smaller parallel work unit sizes
* Allow setting a workspace in config and calling asgen on a config file
* Add backend interface for processing a single package
* Allow calling only the metadata publication step
* Add framework for extracting data directly from package files
* Allow setting icon policy in configuration
* Add support for more icon types and customizable icon policy
* Allow icon upscaling in certain cases, but complain about it
* Modernize Meson definitions
* Document how to modify icon policies in configuration
* Use AppStream 0.12.0 and format 0.12 by default
Bugfixes:
* Update wrap files to be compatible with gir-to-d >= 0.13
* Improve ArchiveDecompressor usage
* For each array appender, try to guess capacity to reserve better
* Properly add remote icon URLs
* Fetch JavaScript bits with Yarn instead of Bower
* Ensure we can run asgen again from its build directory
* Ignore data from immutable suites on cleanup
* Don't parallelize cleanup data collection routine
* Release memory chunks faster during cleanup operations
Version 0.6.8
~~~~~~~~~~~~~~
Released: 2017-11-06
Notes:
* The -Ddownload_js build flag is now -Ddownload-js to follow the naming
of other Meson flags more closely.
Features:
* Encode AppStream library version in version info string on reports
* Don't require an install candidate for a webapp
* Update hicolor theme fallback definition
* Try to filter out symbolic icons for apps
* Allow processing only one section in a suite
* Relax icon scaling rules
Bugfixes:
* Move time graph legend to the top left
* ci: Use the packaged gir-to-d
* Drop an unused gdc conditional
* Ensure the validator never tests web URLs for validity
* Drop LLVM DC bug workaround and allow cross-module inlining
* Make D GIR interface build work again with recent Meson versions
* Don't fail if we have a dupe ID with no pkgname, can happen now with webapps
Version 0.6.7
~~~~~~~~~~~~~~
Released: 2017-10-02
Features:
* ci: Enable gdc again for testing (Matthias Klumpp)
* Make rpmmd backend work (Matthias Klumpp)
* Make an empty main function for embedded unittests (Matthias Klumpp)
* Add a metainfo file (Matthias Klumpp)
* Add a manual page (Matthias Klumpp)
* Sort suite names on the index page (Matthias Klumpp)
Bugfixes:
* use file.exists to check file existence (Antonio Rojas)
* arch: Don't ref generator results (Matthias Klumpp)
* Improve the icon tarball generation code (Matthias Klumpp)
* Resolve deprecation warning (Matthias Klumpp)
Version 0.6.6
~~~~~~~~~~~~~~
Released: 2017-09-22
Features:
* Add basic HiDPI support (Corentin Noël)
* Generate the HiDPI tarball (Corentin Noël)
* Add an 'info' command to dump package information (Matthias Klumpp)
* Handle the "Hidden" property in .desktop files and complain about it (Matthias Klumpp)
Bugfixes:
* arch: XML-escape package descriptions (Matthias Klumpp)
* Check for availability of a component-id at the right time (Matthias Klumpp)
* Make writing compressed results a bit more robust (Matthias Klumpp)
* Resolve all deprecation warnings and slightly improve code (Matthias Klumpp)
Version 0.6.5
~~~~~~~~~~~~~~
Released: 2017-07-02
Features:
* Generate type=codec metadata for gstreamer packages (Iain Lane)
* Add a feature flag for GStreamer processing (Iain Lane)
* ci: Switch to using upstream GirToD (Matthias Klumpp)
* Set asgen version in defines.d by Meson as well (Matthias Klumpp)
* Format graphs a bit nicer (Matthias Klumpp)
Bugfixes:
* Find files in DATADIR - not relative to the executable (Iain Lane)
* Fix typo in metadata-path hint (Matthias Klumpp)
* Try to be more informative when a component has no ID (Matthias Klumpp)
* Ensure the rDNS scheme is really followed before editing an ID (Matthias Klumpp)
Version 0.6.4
~~~~~~~~~~~~~~
Released: 2017-05-25
Features:
* Automatically generate GIR D bindings at build-time (Matthias Klumpp)
* Remove dub build file (Matthias Klumpp)
* Use Meson subproject/wrap to fetch mustache-d in case it is missing (Matthias Klumpp)
* Add a launchable tag to output if we can add one (Matthias Klumpp)
* Make legacy metainfo dir a warning (Matthias Klumpp)
* Validate launchable entries and add heuristics in case they aren't present (Matthias Klumpp)
Bugfixes:
* Import buildPath directly from std.path (Antonio Rojas)
* Remove spurious std.stream import (Matthias Klumpp)
Version 0.6.3
~~~~~~~~~~~~~~
Released: 2017-03-08
Features:
* Experiment with scoped classes (Matthias Klumpp)
* Drop embedded generator copy (Matthias Klumpp)
* Use final classes whenever possible (Matthias Klumpp)
* Centralize all export-dir locations in base config class (Matthias Klumpp)
* Allow to manually configure data export locations (Matthias Klumpp)
Bugfixes:
* Fix some quirks and make the tests work again (Matthias Klumpp)
* ci: Don't build with GDC (Matthias Klumpp)
* Don't fail if a suite has no base suite (Matthias Klumpp)
* engine: Process base suite packages for contents only (Iain Lane)
* Update README.md (#39) (Blake Kostner)
* debian: Only select the most recent packages for a scan (Matthias Klumpp)
* Streamline final component validity check, resolve a crash (Matthias Klumpp)
Version 0.6.2
~~~~~~~~~~~~~~
Released: 2017-01-24
Notes:
* GDC 6.x can't compile the project at time due to a GDC bug.
See https://bugzilla.gdcproject.org/show_bug.cgi?id=251 for more
information.
Features:
* Support desktop-app metainfo files without .desktop file if they have an icon set
* Write log entry when starting/finishing icon tarball write
* Update AppStream bindings
* Allow to specify allowed keys
* Don't show pedantic validator hints
* Allow metainfo file to specify sample texts for fonts
* Process XPM icons if they are large enough
Bugfixes:
* Safeguard against TLD checks with empty-or-null string
* Ensure desktop-apps have at least one valid category set
* Don't override metainfo name/summary with .desktop values
* Quit immediately if the component type is unknown
* Correctly read a font's full-name
* Don't needlessly reprocess fonts
Version 0.6.1
~~~~~~~~~~~~~~
Released: 2016-12-26
Features:
* Add a symlink so that Ubuntu uses the same template as Debian (Iain Lane)
* Support linking to "old suites" in the HTML index (Iain Lane)
* hints: icon-not-found: Explain the symlink problem for Debian & Ubuntu (Iain Lane)
* Add preliminary new asgen logo (Matthias Klumpp)
* Use struct for archive (Matthias Klumpp)
* Reuse AsMetadata while processing metainfo files (Matthias Klumpp)
Bugfixes:
* Correctly install templates with Meson (Matthias Klumpp)
* meson: Make finding mustache-d more robust (Matthias Klumpp)
* Fix build with DMD (Antonio Rojas)
* engine: Don't clean packages in base suites (Iain Lane)
* download: If we have a last-modified date, set the mtime of the target file (Iain Lane)
* Simplify zarchive code a little (Matthias Klumpp)
* Use module initializer to initialize global static data (Matthias Klumpp)
* Fix build on Debian with GDC (Matthias Klumpp)
Version 0.6.0
~~~~~~~~~~~~~~
Released: 2016-10-03
Notes:
* If possible, you should build Meson with the LLVM D compiler and the
Meson build system now.
Features:
* Produce better error messages on failed libarchive actions (Matthias Klumpp)
* Add a hook to the desktop file parser to run backend specific code (Iain Lane)
* Add an Ubuntu backend to retrieve langpack translations (Iain Lane)
* Open databases with NOTLS (Matthias Klumpp)
* Tweak the issue messages a bit (Matthias Klumpp)
* Complain about stuff in legacy paths (Matthias Klumpp)
* Only search for .desktop files if we have a DESKTOP_APP component (Matthias Klumpp)
* Create new Font class to read font metadata (Matthias Klumpp)
* Render an icon for fonts (Matthias Klumpp)
* Add dependencies on Pango, FreeType and Fontconfig (Matthias Klumpp)
* Determine languages a font supports (Matthias Klumpp)
* Reorganize code for proper namespacing (Matthias Klumpp)
* Render font screenshots (Matthias Klumpp)
* Map font full names to files in a spec-compliant way (Matthias Klumpp)
* Enable fonts support by default (Matthias Klumpp)
* Share ContentsStore between threads again (Matthias Klumpp)
* Make Meson a first-class buildsystem for asgen (Matthias Klumpp)
* Update README (Matthias Klumpp)
Bugfixes:
* Fix build on non-64-bit architectures (Matthias Klumpp)
* Work around LDC bug (Matthias Klumpp)
* fclose() the memstream when we are done with it (Iain Lane)
* debpkgindex: Be less noisy about translations in debug mode (Iain Lane)
* Use getTestSamplesDir (Iain Lane)
* Retry on all curl errors, not just timeouts (Iain Lane)
* Synchronise downloading of files (Iain Lane)
* ubuntu: Don't reference all packages, just the ones we need (langpacks) (Iain Lane)
* Use an ugly mutex to work around Fontconfig issues (Matthias Klumpp)
* Make FC mutex a bit more fine-grained (Matthias Klumpp)
Version 0.5.0
~~~~~~~~~~~~~~
Released: 2016-08-30
Features:
* Add more speed and style optimizations, make Package an abstract class (Matthias Klumpp)
* Use a string appender instead of a string array for writing output (Matthias Klumpp)
* debian: Support downloading files from a mirror, instead of having them local (Iain Lane)
* debian: Support DDTP translations (Iain Lane)
* Ignore some more useless categories (Matthias Klumpp)
* Add skeleton for RPMMD (Matthias Klumpp)
* Add a few optimizations using immutable and appender (Matthias Klumpp)
* Do not spawn compressors anymore, use libarchive directly (Matthias Klumpp)
* Make compiling with LDC easier (Matthias Klumpp)
* Add example for cruft collection script (Matthias Klumpp)
* Allow cleaning up statistical data retroactively (Matthias Klumpp)
* Rehash hash tables that we query often (Matthias Klumpp)
* Always optimize debug builds (Matthias Klumpp)
* Make a splitbuild with Meson and Ninja possible (Matthias Klumpp)
* Use AppStream's knowledge about TLDs to build smarter global-component-ids (Matthias Klumpp)
* Allow specifying the format version metadata should be built for (Matthias Klumpp)
* Adjust for AppStream 0.10 release (Matthias Klumpp)
* Perform legal checks before allowing metadata to be added to the pool (Matthias Klumpp)
Bugfixes:
* Fix dataUseJSTime in section overview template (Blake Kostner)
* debian: Only retrieve DDTP translations from the section we're looking at (Iain Lane)
* Fix deprecation warnings from LDC (Matthias Klumpp)
* Work around some weird JSON parser quirk (int incompatible with uint) (Matthias Klumpp)
* Prevent creating excess statistic entries (Matthias Klumpp)
* Fix potential crash (Matthias Klumpp)
* Use native generator when not compiling with GDC (Matthias Klumpp)
* Work correctly if CIDs omit the .desktop suffix (Matthias Klumpp)
* Only add valid category names to output (Matthias Klumpp)
* Don't emit description-from-package multiple times per package. (Matthias Klumpp)
Version 0.4.0
~~~~~~~~~~~~~~
Released: 2016-07-12
Features:
* README: Add link to D tour
* Run optipng with standard settings
* Add command to forget extracted data for a package
* Implement immutable suites
* Use appender more often and apply some style fixes
* Build with parallel by default when using Makefile
* Drop some deprecated stuff
Bugfixes:
* Display an error when processing a suite without section or arch
* debian: Fix exception message if directory does not exist
* Code improvements: More pure, more safe, more trusted
* Drop global data validation result box
* Don't crash if media pool directory does not exist and immutable suites are used
Version 0.3.0
~~~~~~~~~~~~~~
Released: 2016-05-24
Features:
* Register new backend for Arch Linux (Matthias Klumpp)
* arch: Add reader for lists index file and skeleton for PackageIndex (Matthias Klumpp)
* Make generic archive class work well with non-Debian packages (Matthias Klumpp)
* Add HTML anchors for maintainers on report pages (Matthias Klumpp)
* html: Show suite name in section overview (Matthias Klumpp)
* Allow not setting a MediaBaseUrl (Matthias Klumpp)
* Add some font rendering experiments (Matthias Klumpp)
* Allow storing multiple statitics entries per point in time (Matthias Klumpp)
* Make icon tarball build reproducible (Matthias Klumpp)
* Make ArchiveDecompressor API more powerful and always return const(ubyte)[] (Matthias Klumpp)
* arch: Speed up backend by loading all data in one go (Matthias Klumpp)
* Split "handleScreenshots" into more useful, fine-grained flags (Matthias Klumpp)
* Add timestamp to output and only touch it if something has changed (Matthias Klumpp)
* Only update metadata if the indices have been changed (Matthias Klumpp)
* Store media in pool subdirectory (Matthias Klumpp)
* Add flag to enforce metadata processing, even if nothing changed (Matthias Klumpp)
* Allow disabling metadata timestamps (Matthias Klumpp)
* Improve scan-skipping code (Matthias Klumpp)
Bugfixes:
* Fix build on non-64bit arches (Matthias Klumpp)
* Handle compressed empty files correctly (Neil Mayhew)
* Drop other, non-UTF-8 encodings from language codes too (Matthias Klumpp)
* Drop non-printable characters from .desktop file values (Matthias Klumpp)
* Loop over more things by-reference to reduce RAM usage (Matthias Klumpp)
* debian: Read Packages.xz files if no .gz file is available (Matthias Klumpp)
* Don't lie about thumbnail sizes in their filename (Matthias Klumpp)
* Don't duplicate architecture tag if there are multiple issues (Matthias Klumpp)
* Catch data serialization errors (Matthias Klumpp)
* Run bower with allow root to be able to make js as root (Harald Sitter)
Version 0.2.0
~~~~~~~~~~~~~~
Released: 2016-04-24
Features:
* Don't ship minified JS
* Get rid of (almost) all the embedded JS copies, use Bower
* Add makefile for convenience
* Use Flot for drawing graphs
* Update README
* Document all asgen-config settings
* Reserve subdb in contents database to cache icon data
* Make IconHandler use a pre-filtered list of icons
* Allow templates to override only parts of the default branding
* Add some default branding for Debian
* Show logo on generated HTML pages
* debian: Link some interesting resources from the main page
Bugfixes:
* Find aliased icons correctly
* Keep priority sorting of themes
* Make the hicolor theme always-available, using an embedded index copy if necessary
* Fix counting of issues
* Do not accidentally upscale screenshots while creating thumbnails
* debian: Don't make a missing package index fatal
* Don't fail if we are trying to add statistics too quickly
* Rename remove-valid to reflect what it actually does
* Ignore errors if we are trying to decompress a 0-byte gzip file
* Suite arguments aren't optional for process/remove-found
* Demote screenshot-no-thumbnails to info
Version 0.1.0
~~~~~~~~~~~~~~
Released: 2016-04-18
Notes:
* Initial release
appstream-generator-0.8.6/README.md 0000664 0000000 0000000 00000005601 14160667064 0017000 0 ustar 00root root 0000000 0000000 # AppStream Generator
AppStream is an effort to provide additional metadata and unique IDs for all software available in a Linux system.
This repository contains the server-side of the AppStream infrastructure, a tool to generate metadata from distribution packages. You can find out more about AppStream collection metadata at [Freedesktop](https://www.freedesktop.org/software/appstream/docs/chap-CollectionData.html).
The AppStream generator is currently primarily used by Debian, but is written in a distribution agnostic way. Backends only need to implement [two interfaces](src/asgen/backends/interfaces.d) to to be ready.
If you are looking for the AppStream client-tools, the [AppStream repository](https://github.com/ximion/appstream) is where you want to go.
## Development
### Build dependencies
* LDC[1]
* Meson (>= 0.46) [2]
* GLibD [3]
* AppStream [4]
* libarchive (>= 3.2) [5]
* LMDB [6]
* GirToD [7]
* LibSoup
* Cairo
* GdkPixbuf 2.0
* RSvg 2.0
* FreeType
* Fontconfig
* Pango
* Yarn (optional) [8]
[1]: https://github.com/ldc-developers/ldc/releases
[2]: http://mesonbuild.com/
[3]: https://github.com/gtkd-developers/GlibD
[4]: https://github.com/ximion/appstream
[5]: https://libarchive.org/
[6]: https://symas.com/lmdb/
[7]: https://github.com/gtkd-developers/gir-to-d
[8]: https://yarnpkg.com/
On Debian and derivatives of it, all build requirements can be installed using the following command:
```ShellSession
sudo apt install meson ldc gir-to-d libappstream-dev libsoup2.4-dev libarchive-dev \
libgdk-pixbuf2.0-dev librsvg2-dev libcairo2-dev libfreetype6-dev libfontconfig1-dev \
libpango1.0-dev liblmdb-dev libglibd-2.0-dev
```
### Build instructions
To build the tool with Meson, create a `build` subdirectory, change into it and run `meson .. && ninja` to build.
In summary:
```ShellSession
$ mkdir build && cd build
$ meson -Ddownload-js=true ..
$ ninja
$ sudo ninja install
```
We support several options to be set to influence the build. Change into the build directory and run `mesonconf` to see them all.
You might want to perform an optimized debug build by passing `--buildtype=debugoptimized` to `meson` or just do a release build straight
away with `--buildtype=release` in case you want to use the resulting binaries productively. By default, the build happens without optimizations
which slows down the generator.
## Usage
Take a look at the `docs/` directory in the source tree for information on how to use the generator and write configuration files for it.
## Hacking
Pull-requests and patches are very welcome! If you are new to D, it is highly recommended to take a few minutes to look at the D tour to get a feeling of what the language can do: https://tour.dlang.org/
appstream-generator-0.8.6/RELEASE 0000664 0000000 0000000 00000001761 14160667064 0016527 0 ustar 00root root 0000000 0000000 AppStream Generator Release Notes
1. Write NEWS entries for AppStream Generator in the same format as usual.
git shortlog v0.8.5.. | grep -i -v trivial | grep -v Merge > NEWS.new
--------------------------------------------------------------------------------
Version 0.8.6
~~~~~~~~~~~~~~
Released: 2021-xx-xx
Notes:
Features:
Bugfixes:
Contributors:
--------------------------------------------------------------------------------
2. Commit changes in Git:
git commit -a -m "Release version 0.8.6"
git tag -s -f -m "Release 0.8.6" v0.8.6
git push --tags
git push
3. Do post release version bump in meson.build, RELEASE
4. Commit trivial changes:
git commit -a -m "trivial: post release version bump"
git push
5. Send an email to appstream@lists.freedesktop.org
=================================================
AppStream Generator 0.8.6 released!
Tarballs available here: https://github.com/ximion/appstream-generator/releases
=================================================
appstream-generator-0.8.6/TODO 0000664 0000000 0000000 00000000437 14160667064 0016213 0 ustar 00root root 0000000 0000000 = AppStream Generator TODO List =
=== Known issues ===
* No persistent problems at time :-)
=== Planned Features ===
* Use feature from the upcoming libappstream-compose
=== Whishlist / Random Ideas ===
* Add an icon-cache so we don't render SVG icons in themes multiple times.
appstream-generator-0.8.6/contrib/ 0000775 0000000 0000000 00000000000 14160667064 0017157 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/contrib/cleanup-cruft.sh.example 0000664 0000000 0000000 00000001351 14160667064 0023715 0 ustar 00root root 0000000 0000000 #!/bin/bash
#
# Script cleaning up the AppStream metadata pool and cache.
# This script should be run by a cronjob (e.g. every week).
#
set -e
set -o pipefail
set -u
WORKSPACE_DIR="/srv/appstream/workspace"
# only run one instance of the script
LOCKFILE="$WORKSPACE_DIR/.lock"
cleanup() {
rm -f "$LOCKFILE"
}
if ! lockfile -r8 $LOCKFILE; then
echo "aborting AppStream metadata cleanup because $LOCKFILE has already been locked"
exit 0
fi
trap cleanup 0
# Start logging
logdir="$WORKSPACE_DIR/logs/`date "+%Y/%m"`"
mkdir -p $logdir
NOW=`date "+%d_%H%M"`
LOGFILE="$logdir/${NOW}_cleanup.log"
exec >> "$LOGFILE" 2>&1
cd $WORKSPACE_DIR
# Cleanup superseded data
appstream-generator cleanup
# finish logging
exec > /dev/null 2>&1
appstream-generator-0.8.6/contrib/girwrap/ 0000775 0000000 0000000 00000000000 14160667064 0020632 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/contrib/girwrap/APILookup.txt 0000664 0000000 0000000 00000003251 14160667064 0023177 0 ustar 00root root 0000000 0000000 #
# Licensed under the GNU Lesser General Public License Version 3
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the license, or
# (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library. If not, see .
# define the license to use
license: start
/*
* Licensed under the GNU Lesser General Public License Version 3
*
* This library is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this library. If not, see .
*/
// generated automatically - do not change
license: end
includeComments: y
srcDir: .
lookup: APILookupGdkPixbuf.txt
lookup: APILookupAppStream.txt
lookup: APILookupAppStreamCompose.txt
appstream-generator-0.8.6/contrib/girwrap/APILookupAppStream.txt 0000664 0000000 0000000 00000002604 14160667064 0025015 0 ustar 00root root 0000000 0000000 #
# Licensed under the GNU Lesser General Public License Version 3
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the license, or
# (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library. If not, see .
# must start with wrap
wrap: appstream
file: AppStream-1.0.gir
addAliases: start
public import glib.c.types;
public import gobject.c.types;
public import gio.c.types;
public import core.stdc.stdio;
public import core.stdc.string;
addAliases: end
move: version_string Utils appstreamVersion
move: issue_severity_to_string Utils severityToString
move: issue_severity_from_string Utils severityFromString
move: utils_is_tld Utils isTld
move: utils_is_category_name Utils isCategoryName
move: format_version_to_string Utils
move: format_version_from_string Utils
move: license_is_metadata_license Utils
move: spdx_license_tokenize Utils
move: component_kind_to_string Utils
appstream-generator-0.8.6/contrib/girwrap/APILookupAppStreamCompose.txt 0000664 0000000 0000000 00000003377 14160667064 0026353 0 ustar 00root root 0000000 0000000 #
# Licensed under the GNU Lesser General Public License Version 3
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the license, or
# (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library. If not, see .
# must start with wrap
wrap: ascompose
file: AppStreamCompose-1.0.gir
addAliases: start
public import glib.c.types;
public import gobject.c.types;
public import gio.c.types;
public import appstream.c.types;
public import gdkpixbuf.c.types;
public import core.stdc.stdio;
public import core.stdc.string;
addAliases: end
move: image_format_from_filename Utils
struct: Globals
move: globals_get_tmp_dir Globals getTmpDir
move: globals_get_tmp_dir_create Globals getTmpDirCreate
move: globals_set_tmp_dir Globals setTmpDir
move: globals_get_use_optipng Globals useOptipng
move: globals_set_use_optipng Globals setUseOptipng
move: globals_get_optipng_binary Globals optipngBinary
move: globals_set_optipng_binary Globals setOptipngBinary
move: globals_add_hint_tag Globals addHintTag
move: globals_get_hint_tags Globals getHintTags
move: globals_hint_tag_severity Globals hintTagSeverity
move: globals_hint_tag_explanation Globals hintTagExplanation
struct: Result
class: Result
move: add_hint_v Result addHint
move: add_hint_by_cid_v Result addHintByCid
appstream-generator-0.8.6/contrib/girwrap/APILookupGdkPixbuf.txt 0000664 0000000 0000000 00000016374 14160667064 0025015 0 ustar 00root root 0000000 0000000 # This file is part of gtkD.
#
# gtkD is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# gtkD is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with gtkD; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
#############################################
### Definitions for wrapping Gtk+ ###########
#############################################
# must start with wrap
wrap: gdkpixbuf
file: GdkPixbuf-2.0.gir
file: GdkPixdata-2.0.gir
struct: Pixbuf
array: new_from_inline data data_length
noCode: save_to_bufferv
noCode: ref
noCode: unref
noCode: new_from_resource
noCode: new_from_resource_at_scale
noCode: get_pixels
code: start
/**
* Saves pixbuf to a new buffer in format @type, which is currently "jpeg",
* "tiff", "png", "ico" or "bmp". See gdk_pixbuf_save_to_buffer()
* for more details.
*
* Params:
* buffer = location to receive a pointer to the new buffer.
* bufferSize = location to receive the size of the new buffer.
* type = name of file format.
* optionKeys = name of options to set, %NULL-terminated
* optionValues = values for named options
*
* Return: whether an error was set
*
* Since: 2.4
*
* Throws: GException on failure.
*/
public bool saveToBuffer(out ubyte[] buffer, string type, string[] optionKeys, string[] optionValues)
{
char* outbuffer = null;
size_t bufferSize;
GError* err = null;
auto p = gdk_pixbuf_save_to_bufferv(gdkPixbuf, &outbuffer, &bufferSize, Str.toStringz(type), Str.toStringzArray(optionKeys), Str.toStringzArray(optionValues), &err) != 0;
if (err !is null)
{
throw new GException( new ErrorG(err) );
}
buffer = (cast(ubyte*)outbuffer)[0 .. bufferSize];
return p;
}
/**
* Creates a new pixbuf by loading an image from an resource.
*
* The file format is detected automatically.
*
* Params:
* resourcePath = the path of the resource file
*
* Return: A newly-created pixbuf, or null if any of several error
* conditions occurred: the file could not be opened, the image format is
* not supported, there was not enough memory to allocate the image buffer,
* the stream contained invalid data, or the operation was cancelled.
*
* Since: 2.26
*
* Throws: GException on failure.
*/
public static Pixbuf newFromResource(string resourcePath)
{
GError* err = null;
auto p = gdk_pixbuf_new_from_resource(Str.toStringz(resourcePath), &err);
if (err !is null)
{
throw new GException( new ErrorG(err) );
}
return new Pixbuf(cast(GdkPixbuf*) p, true);
}
/**
* Creates a new pixbuf by loading an image from an resource.
*
* The file format is detected automatically.
*
* The image will be scaled to fit in the requested size, optionally
* preserving the image's aspect ratio. When preserving the aspect ratio,
* a width of -1 will cause the image to be scaled to the exact given
* height, and a height of -1 will cause the image to be scaled to the
* exact given width. When not preserving aspect ratio, a width or
* height of -1 means to not scale the image at all in that dimension.
*
* The stream is not closed.
*
* Params:
* resourcePath = the path of the resource file
* width = The width the image should have or -1 to not constrain the width
* height = The height the image should have or -1 to not constrain the height
* preserveAspectRatio = true to preserve the image's aspect ratio
*
* Return: A newly-created pixbuf, or null if any of several error
* conditions occurred: the file could not be opened, the image format is
* not supported, there was not enough memory to allocate the image buffer,
* the stream contained invalid data, or the operation was cancelled.
*
* Since: 2.26
*
* Throws: GException on failure.
*/
public static Pixbuf newFromResource(string resourcePath, int width, int height, bool preserveAspectRatio)
{
GError* err = null;
auto p = gdk_pixbuf_new_from_resource_at_scale(Str.toStringz(resourcePath), width, height, preserveAspectRatio, &err);
if (err !is null)
{
throw new GException( new ErrorG(err) );
}
return new Pixbuf(cast(GdkPixbuf*) p, true);
}
/**
* Queries a pointer to the pixel data of a pixbuf.
*
* Return: A pointer to the pixbuf's pixel data.
* Please see the section on [image data](image-data) for information
* about how the pixel data is stored in memory.
*
* This function will cause an implicit copy of the pixbuf data if the
* pixbuf was created from read-only data.
*/
public char* getPixels()
{
return gdk_pixbuf_get_pixels(gdkPixbuf);
}
code: end
struct: PixbufAnimation
noCode: new_from_resource
struct: PixbufLoader
import: glib.Str
noCode: new_with_type
noCode: new_with_mime_type
code: start
/**
* Creates a new pixbuf loader object that always attempts to parse
* image data as if it were an image of type @image_type, instead of
* identifying the type automatically. Useful if you want an error if
* the image isn't the expected type, for loading image formats
* that can't be reliably identified by looking at the data, or if
* the user manually forces a specific type.
*
* The list of supported image formats depends on what image loaders
* are installed, but typically "png", "jpeg", "gif", "tiff" and
* "xpm" are among the supported formats. To obtain the full list of
* supported image formats, call gdk_pixbuf_format_get_name() on each
* of the #GdkPixbufFormat structs returned by gdk_pixbuf_get_formats().
*
* Params:
* imageType = name of the image format to be loaded with the image
* isMimeType = Set to true if type is a mime type
*
* Return: A newly-created pixbuf loader.
*
* Throws: GException on failure.
*/
public this(string type, bool isMimeType=false)
{
GError* err = null;
GdkPixbufLoader* p;
if ( isMimeType )
{
p = cast(GdkPixbufLoader*)gdk_pixbuf_loader_new_with_mime_type(Str.toStringz(type), &err);
}
else
{
p = cast(GdkPixbufLoader*)gdk_pixbuf_loader_new_with_type(Str.toStringz(type), &err);
}
if (err !is null)
{
throw new GException( new ErrorG(err) );
}
this(cast(GdkPixbufLoader*) p, true);
}
code: end
struct: PixbufSimpleAnim
class: PixbufSimpleAnimation
struct: PixbufSimpleAnimIter
class: PixbufSimpleAnimationIter
noCode: true
struct: Pixdata
noProperty: pixel_data
move: pixbuf_from_pixdata Pixbuf from_pixdata
# Gir-to-d tries to handle the return value of 'gdk_pixdata_from_pixbuf' as if it is zero-terminated.
# Which causes it to try to use 'getArrayLength' on a void*, causing a compilation error.
# Temporary fix for now is to just not export that function.
noCode: from_pixbuf
# Add missing include for FILE* that GdkPixbufModule needs inside 'girepo/gdkpixbuf/c/types.d'.
addAliases: start
private import core.stdc.stdio : FILE;
addAliases: end
appstream-generator-0.8.6/contrib/setup/ 0000775 0000000 0000000 00000000000 14160667064 0020317 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/contrib/setup/.gitignore 0000664 0000000 0000000 00000000030 14160667064 0022300 0 ustar 00root root 0000000 0000000 node_modules/
yarn.lock
appstream-generator-0.8.6/contrib/setup/build_js.sh 0000775 0000000 0000000 00000001257 14160667064 0022456 0 ustar 00root root 0000000 0000000 #!/bin/sh
set -e
if [ -n "$MESON_SOURCE_ROOT" ]; then
cd "$MESON_SOURCE_ROOT/contrib/setup/"
fi
YARNPKG="yarn"
if [ ! -z "$1" ]
then
YARNPKG=$1
fi
$YARNPKG install --no-bin-links --prod --no-lockfile --non-interactive
JS_TARGET=../../data/templates/default/static/js
[ ! -d "$JS_TARGET" ] && mkdir $JS_TARGET
[ ! -d "$JS_TARGET/jquery" ] && mkdir $JS_TARGET/jquery
install node_modules/jquery/dist/*.min.js -t $JS_TARGET/jquery
[ ! -d "$JS_TARGET/flot" ] && mkdir $JS_TARGET/flot
install node_modules/jquery-flot/jquery.flot*.js -t $JS_TARGET/flot
[ ! -d "$JS_TARGET/highlight" ] && mkdir $JS_TARGET/highlight
install node_modules/highlightjs/*.js -t $JS_TARGET/highlight
appstream-generator-0.8.6/contrib/setup/meson-install-templates.sh 0000775 0000000 0000000 00000000410 14160667064 0025432 0 ustar 00root root 0000000 0000000 #!/bin/sh
set -e
cd "$MESON_SOURCE_ROOT"
echo "Installing templates..."
install -d "${DESTDIR}/${MESON_INSTALL_PREFIX}/share/appstream/templates"
cp -dpru --no-preserve=ownership data/templates/* -t "${DESTDIR}/${MESON_INSTALL_PREFIX}/share/appstream/templates"
appstream-generator-0.8.6/contrib/setup/package.json 0000664 0000000 0000000 00000000176 14160667064 0022611 0 ustar 00root root 0000000 0000000 {
"flat": true,
"dependencies": {
"highlightjs": "^9.10.0",
"jquery": "^3.3.1",
"jquery-flot": "^0.8.3"
}
}
appstream-generator-0.8.6/contrib/update-metadata.sh.example 0000664 0000000 0000000 00000001715 14160667064 0024211 0 ustar 00root root 0000000 0000000 #!/bin/bash
#
# Example script for updating AppStream metadata using appstream-generator.
# This script can easily be run by a cronjob.
#
set -e
set -o pipefail
set -u
SUITES="sid stretch"
WORKSPACE_DIR="/srv/appstream/workspace"
PUBLIC_DIR="/srv/appstream/public"
# only run one instance of the script
LOCKFILE="$WORKSPACE_DIR/.lock"
cleanup() {
rm -f "$LOCKFILE"
}
if ! lockfile -r8 $LOCKFILE; then
echo "aborting AppStream metadata extraction because $LOCKFILE has already been locked"
exit 0
fi
trap cleanup 0
# Start logging
logdir="$WORKSPACE_DIR/logs/`date "+%Y/%m"`"
mkdir -p $logdir
NOW=`date "+%d_%H%M"`
LOGFILE="$logdir/${NOW}.log"
exec >> "$LOGFILE" 2>&1
cd $WORKSPACE_DIR
# generate fresh metadata
for suite in $SUITES; do
appstream-generator process $suite
done
# Sync updated data to public directory
rsync -ak --delete-after --link-dest="$PUBLIC_DIR/" "$WORKSPACE_DIR/export/" "$PUBLIC_DIR/"
# finish logging
exec > /dev/null 2>&1
appstream-generator-0.8.6/data/ 0000775 0000000 0000000 00000000000 14160667064 0016430 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/data/.gitignore 0000664 0000000 0000000 00000000155 14160667064 0020421 0 ustar 00root root 0000000 0000000 templates/default/static/js/jquery/
templates/default/static/js/flot/
templates/default/static/js/highlight/
appstream-generator-0.8.6/data/asgen-hints.json 0000664 0000000 0000000 00000015356 14160667064 0021555 0 ustar 00root root 0000000 0000000 {
"internal-error": {
"text": [ "A fatal problem appeared in the generator.",
"Please report a bug: {{msg}}"],
"severity": "error"
},
"internal-unknown-tag": {
"text": ["The generator emitted a tag '{{tag}}' which is unknown. This is a bug in the metadata generator, please",
"file a bugreport."],
"severity": "warning"
},
"icon-format-unsupported": {
"text": "Icon file '{{icon_fname}}' uses an unsupported image file format.",
"severity": "error"
},
"icon-not-found": {
"text": [ "The icon '{{icon_fname}}' was not found in the archive. This issue can have multiple reasons:",
"
",
"
The icon is not present in the archive.
",
"
The icon is in a wrong directory.
",
"
The icon is not available in a suitable size (at least 64x64px)
",
"
On Debian and Ubuntu, the icon is a symlink. The generator cannot read symlinks on these distributions - make the icon a real file.
AppStream is a metadata specification which permits software components to provide information about themselves
to automated systems and end-users before the software is actually installed.
The appstream-generator tool generates AppStream metadata from the repositories of a software distribution.
It currently supports the following repository formats / distributions: Debian, Ubuntu, Arch Linux, RPM-MD (Fedora, Mageia).
The generator will produce AppStream collection metadata files in the AppStream YAML or XML format to be shipped
to users, as well as a detailed HTML report about found components and HTML and JSON reports on issues detected
with the scanned metadata. It reads .desktop files as well as metainfo files, renders fonts, scales images, caches
screenshots etc. to produce high-quality metadata for AppStream based software centers to consume.
Usually, appstream-generator is integrated with the existing software build & packaging workflow of
a distribution.
These pages exist to provide a user-friendly view on the issues discovered by the
AppStream metadata generator while extracting metadata from packages in the {{project_name}} archive.
They can also be used to take a look at the raw metadata, to spot possible problems
with the data itself or the generation process.
AppStream is a cross-distro XML format to provide metadata for software components and to assign unique identifiers to software.
In Debian, we parse all XML provided by upstream projects as well as other metadata (.desktop-files, ...), and compile a single YAML
metadata file from it, which is then shipped to users via APT.
While the official AppStream specification is based on XML, Debian uses a YAML version of the format for easier use in existing
scripts and for better archive integration. This format is called DEP-11, and initially had a much wider scope in enhancing
archive metadata than AppStream had. Today AppStream covers that as well, and DEP-11 is only a YAML version of AppStream.
The generated metadata can for example be used by software centers like GNOME Software or KDE Discover to display a user-friendly application-centric
view on the package archive.
It can also be used by other software to find missing plugins, codecs, fonts, etc. or simply by users to install software on any Linux distribution
without knowing the exact package name.
More information
See AppStream @ wiki.d.o for information about AppStream integration and usage in Debian.
The offical AppStream specification can be found at freedesktop.org, a description
of the DEP-11 YAML format is hosted there as well.
You can find the source-code of the AppStream Generator here.
Log files of the generator runs are stored in logs/, machine-readable issue hints can be found in hints/,
valid metadata is located in data/ and all exported media is made available via media/.
These pages exist to provide a user-friendly view on the issues discovered by the
AppStream metadata generator while extracting metadata from packages in the {{project_name}} archive.
They can also be used to take a look at the raw metadata, to spot possible problems
with the data itself or the generation process.
AppStream is a cross-distro XML format to provide metadata for software components and to assign unique identifiers to software.
In {{project_name}}, we parse all XML provided by upstream projects as well as other metadata (.desktop-files, ...), and compile a single
metadata file from it, which is then shipped to users.
The generated metadata can for example be used by software centers like GNOME Software or KDE Discover to display a user-friendly application-centric
view on the package archive.
It can also be used by other software to find missing plugins, codecs, fonts, etc. or simply by users to install software on any Linux distribution
without knowing the exact package name.
More information
The offical AppStream specification can be found at freedesktop.org.
You can find the source-code of the AppStream Generator here.
X7%9]`Җ'l$Qn`2M+ǣU 4sLTV(cnCi%+" Jma?
|S>$WI2KOXk 8( }" "$E EȩPǿ OYB- 3W '@Jc1NğYCjUi,h jO]I$^A~C<Aq4ha_ #G|< QOU)#r {X:a-|m+!qU8i ;[-x-xl-U)b)hfY͢_)"!dƒ} IENDB` appstream-generator-0.8.6/data/templates/ubuntu 0000777 0000000 0000000 00000000000 14160667064 0023033 2debian ustar 00root root 0000000 0000000 appstream-generator-0.8.6/docs/ 0000775 0000000 0000000 00000000000 14160667064 0016447 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/docs/appstream-generator.1.xml 0000664 0000000 0000000 00000017042 14160667064 0023314 0 ustar 00root root 0000000 0000000
18 April,2016">
GNU">
GPL">
]>
appstream-generator2016-2018Matthias KlumppAppStream Generator
&date;
appstream-generator1&package;Generate AppStream metadata from distribution repositories&package;Description
This manual page documents briefly the &package; command.
&package; generates AppStream metadata from the repositories of a software distribution.
It currently supports the following repository formats / distributions: Debian, Ubuntu, Arch Linux, RPM-MD (Fedora,
Mageia).
The generator will produce AppStream collection metadata files in the AppStream YAML or XML format to be shipped
to users, as well as detailed HTML reports about found components and HTML and JSON reports on issues found
while compiling the metadata. It reads .desktop files as well as metainfo files, renders fonts, scales images, caches
screenshots etc. to produce high-quality metadata for AppStream based software centers and other tools to consume.
Usually, &package; is integrated with the existing software build & delivery workflow of
a distribution.
The &package; tool is based on the libappstream library for metadata conversion and analysis.
If you just want to embed AppStream metadata processing into another tool, using libappstream directly is likely a
better choice. The generator tool does some heavy lifting like rendering fonts and scaling images, which might not be necessary
for simple cases.
To use &package;, a asgen-config.json file is required. Its format is described in detail
in the asgen-config.json documentation.
The generator supports a wide range of features that can individually configured to fit the needs of different projects and adjust the generated
metadata to specific use cases. Refer to the configuration file documentation for information on what options are available.
For more information about the AppStream project and the other components which are part of it, take a look at
the AppStream pages at Freedesktop.org.
Options
Process new metadata for the given distribution suite and publish it.
Cleanup old/expired metadata and media files from the cache and directories.
It is recommended to run this command every week, or at least every month, depending
on how many changes happen in the software repository.
Export all metadata and publish reports in the export directories.
You usually do not want to run this task explicitly, because it is already automatically
performed by the command.
Drop all valid processed metadata and hints from the selected suite.
Drop all information we have about this (partial) package-id.
A package-id consists of a name/version/arch triplet.
For this command, the version and architecture can be omitted to forget
all packages that match a particular name or name-version combination.
Show information associated with this (full) package-id.
A package-id consists of a name/version/arch triplet.
Define the workspace location.
If this flag is omitted, and no workspace directory is given in the generator
configuration file, the current directory is assumed as the workspace location.
This parameter, if given, overrides any workspace location defined elsewhere.
Define a configuration file.
Explicitly set a generator configuration JSON file. If this flag is omitted, the asgen-config.json file
in the current workspace directory is used.
If no workspace directory is defined in the configuration file itself, the directory it is located in is used
as workspace. This can be overridden by defining a workspace explicitly with .
Enforce the command.Show extra debugging information.Display the version number of &package;.See Alsoappstreamcli (1).AUTHOR
This manual page was written by Matthias Klumpp matthias@tenstral.net.
appstream-generator-0.8.6/docs/asgen-config.md 0000664 0000000 0000000 00000025335 14160667064 0021341 0 ustar 00root root 0000000 0000000 # Generator Project Configuration
This document describes the options and fields which can be set in an `asgen-config.json` file.
## JSON file example
An example `asgen-config.json` file may look like this:
```JSON
{
"ProjectName": "Tanglu",
"ArchiveRoot": "/srv/archive.tanglu.org/tanglu/",
"MediaBaseUrl": "http://metadata.tanglu.org/appstream/media",
"HtmlBaseUrl": "http://metadata.tanglu.org/appstream/",
"Backend": "debian",
"Features":
{
"validateMetainfo": true
},
"Suites":
{
"chromodoris":
{
"sections": ["main", "contrib"],
"architectures": ["amd64", "i386"]
},
"chromodoris-updates":
{
"dataPriority": 10,
"baseSuite": "chromodoris",
"sections": ["main", "contrib"],
"architectures": ["amd64", "i386"]
}
},
"Icons":
{
"64x64": {"cached": true, "remote": false},
"128x128": {"cached": false, "remote": true}
}
}
```
## Description of fields
### Toplevel fields
Key | Comment
------------ | -------------
ProjectName | The name of your project or distribution which ships AppStream metadata.
Backend | The backend that should be used to obtain the raw data. Defaults to `debian` if not set.
MetadataType | The type of the resulting AppStream metadata. Can be one of `YAML` or `XML`. If omitted, the backend's default value is used.
ArchiveRoot | A local URL to the mirror of your archive, containing the dists/ and pool/ directories
MediaBaseUrl | The http or https URL which should be used in the generated metadata to fetch media like screenshots or icons
HtmlBaseUrl | The http or https URL to the web location where the HTML hints will be published. (This setting is optional, but recommended)
Oldsuites | This key exists to support migration from an alternative appstream generator. Given a list of suite names, the output HTML will link to `suitename/index.html`.
Suites | Suites which should be recognized by the generator. Each suite has the components and architectures which should be searched for metadata as children. See below for more information.
Features | Disable or enable selected generator features. For a detailed description see below.
CAInfo | Set the CA certificate bundle file to use for SSL peer verification. If this is not set, the generator will use the system default.
AllowedCustomKeys | Set which keys of the tag are allowed to be propagated to the collection metadata output. This key takes a list of custom-key strings as value.
ExportDirs | Set where to export data. The dictionary requires full paths set for the "Media", "Data", "Hints" or "Html" key. In case a value is missing, the default locations are used.
ExtraMetainfoDir | Path to a directory where additional injected metainfo files are located. If not set, the `extra-metainfo` directory in the project workspace is used.
WorkspaceDir | Explicitly set the location of the workspace. Only makes sense if the generator is meant to be used with a lot of configuration files and the configuration is passed to it via the `-c` flag.
Icons | Customize the icon policy. See below for more details.
MaxScreenshotFileSize | The maximum size of downloaded screenshot image or video files in MiB. `0` means unlimited. *Default: `14`*
### Suite fields
The `Suites` field contains a dictionary of the suites which should be processed as value. These suites can contain a selection of properties:
Key | Comment
------------ | -------------
sections | A list of sections the suite possesses. The "sections" are also known as archive components in the Debian world. *(required)*
architectures | A list of architectures which should be processed for this suite. *(required)*
baseSuite | An optional base suite name which should be used in addition to the child suite to resolve icons (only the `main` section of that suite is considered).
dataPriority | An integer value representing the priority the data generated for this suite should have. Metadata with a higher priority will override existing data (think of an `-updates` suite wanting to override data shipped with the base suite). If this is not set, AppStream client tools will assume the priority being `0`.
useIconTheme | Set a specific icon theme name with highest priority for this suite. This is useful if you want a different default icon theme providing icons for generic icon names (by default, the default themes of KDE and GNOME are used).
immutable | If set to `true`, the state of the metadata files and exported data will be frozen, and no more changes to the data for this suite will be allowed. This only works if the `immutableSuites` feature is enabled.
### Enabling and disabling features
Several features of the metadata generator can be toggled to make it work in different scenarios.
The following feature values are recognized, and can be enabled or disabled in the JSON document. If no explicit value is set for a feature, the generator will pick its default value, which is sane in most cases.
Name | Comment
------------ | -------------
validateMetainfo | Validate the AppStream upstream metadata. The validation is slow, but will produce better feedback and issue hints if enabled. *Default: `ON`*
processDesktop | Process .desktop files which do not have a metainfo file. If disabled, all data without metainfo file will be ignored. *Default: `ON`*
noDownloads | Do not attempt any downloads. This will implicitly disable any handling of screenshots and possibly other features. Using this flag is discouraged. *Default: `OFF`*
createScreenshotsStore | Mirror screenshots and create thumbnails of them in `media/`. This will yield the best experience with software-centers, and also allow full control over which screenshots are displayed. Disabling this will make clients pull screenshots from 3rd-party upstream servers. *Default: `ON`*
optimizePNGSize | Use `optipng` to reduce the size of PNG images. Optipng needs to be installed. *Default: `ON`*
metadataTimestamps | Write timestamps into generated metadata files. *Default: `ON`*
immutableSuites | Allow suites to be marked as immutable. This is useful for distributions with fixed releases, but not for rolling release distributions or continuously updated repositories. *Default: `ON`*
processFonts | Include font metadata and render fonts. *Default: `ON`*
allowIconUpscaling | Allows upscaling of small 48x48px icons to 64x64px to make applications show up. Icons are only upscaled as a last resort. *Default: `ON`*
processGStreamer | Synthesise `type=codec` metadata from available GStreamer packages. Requires support in the backend, currently only implemented for Debian. *Default: `ON`*
processLocale | Try to extract the software's localization status from Gettext data. *Default: `ON`*
screenshotVideos | Permit videos in screenshots and cache them if downloads are permitted. *Default: `ON`*
propagateMetaInfoArtifacts | Release artifact information is filtered out by default if a package is set for the selected metadata. Set this flag to propagate artifact information unconditionally. *Default: `OFF`*
### Configuring icon policies
The `Icons` field allows to customize the icon policy used for a generator run. It decides which icon sizes are extracted, and whether they are stored as cached icon, remote icons or both.
The field contains a dictionary with icon sizes as keys. Valid icon sizes are `48x48`, `64x64` and `128x128` and their HiDPI variants (e.g. `64x64@2`).
The values for the icon-size keys are dictionaries with two boolean keys, `cached` and `remote`, to select the storage method for the icon size.
Cached means an icon tarball is generated for the icon size that can be made available locally, while remote means the icon can be downloaded on-demand by the software center and no local cache of all icons exists. Icon sizes not mentioned, or with both `cached` and `remote` set to `false` will not be extracted.
The `64x64` icon size must always be present and be cached. If this is not the case, appstream-generator will adjust the configuration internally and emit a warning.
If no `Icons` field is present, appstream-generator will use a default policy for icons (creating cache tarballs for all sizes, and remote links for sizes >= 129x128px).
## Injecting extra metainfo / removing components
Sometimes injecting metainfo files directly into the generation process instead of packaging them makes sense. This can be done for example for `web-application` components, `operating-system` components or for components which are merged into others. It is discouraged to use this feature for any components that are directly tied to a package.
Metainfo XML files can be placed in `%ExtraMetainfoDir%/suite/section/(arch)`, where `%ExtraMetainfoDir%` usually is the `extra-metainfo` directory in the generator's current workspace, unless this default is overriden in the configuration file.
If the `arch` part of the path is omitted, a metainfo file is added to all active architectures. E.g. a XML file placed in `%ExtraMetainfoDir%/buster/main/` will apply to all active architectures for `buster`, while files in `%ExtraMetainfoDir%/buster/main/amd64/` will only apply to the `amd64` architecture.
Additionally to metainfo XML files, a `removed-components.json` file may be placed in the extra-metainfo directories. This JSON file contains a list of component IDs that should be removed from the metadata pool on client machines. By listing IDs in this file, the generator will create a special component that will trigger a component with the same ID to be removed from the metadata pool if its priority is lower.
This feature is useful for overlay suites, for example if the `buster-updates` suite (with priority 10) wants to completely remove a component from the `buster` suite (with priority 0) from the user's eyes. This may be necessary if a component improperly changes its ID, if you are maintaining an overlay distribution and can not control the composition of packages in the base suites, or in case of legal issues where a component has to be removed retroactively from a frozen distribution suite. It may also be useful to hide `web-application` components in a stable distribution in case their service is discontinued before the distribution release is out of support as well.
Internally, injected metainfo data is grouped under a special fake package name, in order to allow the generator to properly record hints for the added components and to associate the data with the right suite(s).
## Minimal configuration file
A minimal configuration file can look like this:
```JSON
{
"ProjectName": "Tanglu",
"ArchiveRoot": "/srv/archive.tanglu.org/tanglu/",
"MediaBaseUrl": "http://metadata.tanglu.org/appstream/media",
"HtmlBaseUrl": "http://metadata.tanglu.org/appstream/",
"Backend": "debian",
"Suites":
{
"chromodoris":
{
"sections": ["main", "contrib"],
"architectures": ["amd64", "i386"]
}
}
}
```
appstream-generator-0.8.6/docs/index.md 0000664 0000000 0000000 00000000227 14160667064 0020101 0 ustar 00root root 0000000 0000000 # AppStream Generator Docs
For general information check out the [README file](../README.md)
See [usage.md](usage.md) for general usage information.
appstream-generator-0.8.6/docs/meson.build 0000664 0000000 0000000 00000001205 14160667064 0020607 0 ustar 00root root 0000000 0000000 # Meson definition for AppStream Generator Documentation
# make manual page
xsltproc = find_program('xsltproc')
custom_target('man-asgen',
input: 'appstream-generator.1.xml',
output: 'appstream-generator.1',
install: true,
install_dir: join_paths(get_option('mandir'), 'man1'),
command: [
xsltproc,
'--nonet',
'--stringparam', 'man.output.quietly', '1',
'--stringparam', 'funcsynopsis.style', 'ansi',
'--stringparam', 'man.th.extra1.suppress', '1',
'-o', '@OUTPUT@',
'http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl',
'@INPUT@'
]
)
appstream-generator-0.8.6/docs/usage.md 0000664 0000000 0000000 00000004565 14160667064 0020107 0 ustar 00root root 0000000 0000000 # AppStream Generator Usage
## How to use
### Generating distro metadata
To generate AppStream distribution metadata for your repository, create a local
mirror of the repository first.
Then create a new folder, and write a `asgen-config.json` configuration file for the
metadata generator. Details on the file and an example can be found in [the asgen-config docs](asgen-config.md).
After the config file has been written, you can generate the metadata as follows:
```Bash
cd /srv/asgen/workspace # path where the asgen-config.json file is located
appstream-generator process chromodoris # replace "chromodoris" with the name of the suite you want to analyze
```
The generator is assuming you have enough memory and disk space on your machine to cache stuff.
Resulting metadata will be placed in `export/data/`, machine-readable issue-hints can be found in `export/hints/` and the processed screenshots and icons are located in `export/media/`.
In order to drop old packages and cruft from the databases, you should run
```Bash
appstream-generator cleanup
```
every once in a while. This will drop all superseded packages and data from the caches.
If you do not want to `cd` into the workspace directory, you can also use the `--workspace|-w` flag to define a workspace.
### Validating metadata
You can validate the resulting metadata using the AppStream client tools.
Use `appstreamcli validate .xml.gz` for XML metadata, and `dep11-validate .yml.gz` for YAML. This will check the files for mistakes and compliance with the specification.
Keep in mind that the generator will always generate spec-compliant metadata, but might - depending on the input - produce data which has smaller flaws (e.g. formatting issues in the long descriptions). In these cases, issue-hints will have been emitted, so the package maintainers can address the metadata issues.
## Troubleshooting
### Memory Usage
The `appstream-generator` will not hesitate to use RAM, and also decide to use lots of it if enough is available. This is especially true when scanning new packages for their contents and storing the information in the LMDB database.
Ideally make sure that you are running a 64bit system with at least 4GB of RAM if you want to use the generator properly.
For the generator, speed matters more than RAM usage. You can use cgroups to limit the amount of memory the generator uses.
### Profiling
TODO
appstream-generator-0.8.6/meson.build 0000664 0000000 0000000 00000004336 14160667064 0017667 0 ustar 00root root 0000000 0000000 project('AppStream Generator', 'd',
meson_version : '>=0.46',
subproject_dir : 'contrib/subprojects',
license : 'LGPL-3.0+',
version : '0.8.6'
)
asgen_version = meson.project_version()
source_root = meson.source_root()
build_root = meson.build_root()
#
# Dependencies
#
src_dir = include_directories('src/')
glib_dep = dependency('glib-2.0', version: '>= 2.62')
glibd_dep = dependency('glibd-2.0')
appstream_dep = dependency('appstream', version : '>= 0.14.6')
ascompose_dep = dependency('appstream-compose', version : '>= 0.15.1')
lmdb_dep = dependency('lmdb', version : '>= 0.9.22')
archive_dep = dependency('libarchive', version : '>= 3.2')
curl_dep = dependency('libcurl')
#
# Build interfaces from GIR
#
gir_to_d_prog = find_program('girtod')
gir_wrap_dir = source_root + '/contrib/girwrap/'
gir_d_intf_dir = build_root + '/girepo/'
message('Generating AppStream D interfaces from GIR...')
girtod_gen = run_command(gir_to_d_prog,
'-i', gir_wrap_dir,
'-o', gir_d_intf_dir,
'--print-files', 'relative,' + source_root)
if girtod_gen.returncode() != 0
error('Unable to build D interfaces from GIR:\n' + girtod_gen.stderr())
endif
gir_bind_dir = include_directories('girepo')
gir_binding_sources = girtod_gen.stdout().strip().split('\n')
# static library of bindings automatically generated from GIR
girbind_lib = static_library('girbindings',
[gir_binding_sources],
include_directories: [gir_bind_dir],
dependencies: [glibd_dep]
)
#
# Download JS stuff and additional sources if we couldn't find them
#
if get_option('download-js')
yarn_exe = find_program('/usr/share/yarn/bin/yarn', 'yarnpkg') # check for the yarn executable
message('Downloading JavaScript libraries...')
if run_command('[', '-d', 'data/templates/default/static/js', ']').returncode() != 0
getjs_cmd = run_command([source_root + '/contrib/setup/build_js.sh', yarn_exe])
if getjs_cmd.returncode() != 0
error('Unable to download JavaScript files with Yarn:\n' + getjs_cmd.stdout() + getjs_cmd.stderr())
endif
endif
endif
# asgen sources
subdir('src')
# documentation
subdir('docs')
#data
subdir('data')
appstream-generator-0.8.6/meson_options.txt 0000664 0000000 0000000 00000000300 14160667064 0021145 0 ustar 00root root 0000000 0000000 #
# Options for AppStream Generator
#
option('download-js',
type: 'boolean',
value: true,
description: 'Whether we should download JavaScript with Yarn automatically.'
)
appstream-generator-0.8.6/snapcraft.yaml 0000664 0000000 0000000 00000015660 14160667064 0020374 0 ustar 00root root 0000000 0000000 #
# Copyright (C) 2020 Matthias Klumpp
# Copyright (C) 2020 Canonical Ltd
# Author: Iain Lane
#
# SPDX-License-Identifier: FSFAP
#
# This Snapcraft file and the resulting Snap is NOT OFFICIALLY SUPPORTED.
# If you can, please use a provided native build for your distribution.
# Patches to improve this file are very welcome though, and so is testing the Snap!
#
# This snap is strict, but it has an important limitation:
# - it needs read/write access to wherever the user puts the workspace
# definition (could be anywhere, but most commonly it's in /srv, in /home or
# in /var), access to a local archive mount (can be in /mnt), and that should
# be it (unless optipng and ffprobe have extra requirements).
#
# Until that is overcome, this snap should be installed using `--devmode` to
# effectively disable the confinement.
#
# This Snap is autobuilt, you can check its status at https://build.snapcraft.io/user/ximion/appstream-generator
name: appstream-generator
license: LGPL-3.0
base: core20
adopt-info: appstream-generator
confinement: strict
grade: stable
architectures:
- build-on: [amd64]
run-on: [amd64]
apps:
appstream-generator:
command: usr/bin/appstream-generator
common-id: org.freedesktop.appstream.generator
plugs:
- network
- home
environment:
GSETTINGS_SCHEMA_DIR: "$SNAP/usr/share/glib-2.0/schemas"
GIO_EXTRA_MODULES: "$SNAP/usr/lib/$SNAPCRAFT_ARCH_TRIPLET/gio/modules"
layout:
/usr/share/appstream:
bind: $SNAP/usr/share/appstream
/usr/lib/$SNAPCRAFT_ARCH_TRIPLET/gdk-pixbuf-2.0:
bind: $SNAP/usr/lib/$SNAPCRAFT_ARCH_TRIPLET/gdk-pixbuf-2.0
/usr/share/i18n:
bind: $SNAP/usr/share/i18n
/usr/share/mime:
bind: $SNAP/usr/share/mime
parts:
appstream-generator:
source: .
source-type: git
parse-info: [usr/share/metainfo/org.freedesktop.appstream.generator.metainfo.xml]
override-pull: |
snapcraftctl pull
# set version from Git
snapcraftctl set-version "$(git describe --always | sed -e 's/v//;s/-/+git/;y/-/./')"
# adjust to an absolute path to help finding the GIR file from the AppStream part
sed -i 's|AppStream-1.0.gir|$SNAPCRAFT_STAGE/usr/share/gir-1.0/AppStream-1.0.gir|g' ${SNAPCRAFT_PART_SRC}/contrib/girwrap/APILookupAppStream.txt
sed -i 's|AppStreamCompose-1.0.gir|$SNAPCRAFT_STAGE/usr/share/gir-1.0/AppStreamCompose-1.0.gir|g' ${SNAPCRAFT_PART_SRC}/contrib/girwrap/APILookupAppStreamCompose.txt
plugin: meson
build-environment:
- LD_LIBRARY_PATH: $SNAPCRAFT_STAGE/usr/lib/$SNAPCRAFT_ARCH_TRIPLET/
override-build: |
# install yarn
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
apt-get update && apt-get install --no-install-recommends -y yarn nodejs
# actually build asgen - we need to run everything manually here,
# because snapcraft will kill the build if run with maximum amount of ninja jobs,
# and I found no way to limit the amount of permitted ninja jobs other than overriding everything
meson --prefix=/usr --buildtype=debugoptimized -Ddownload-js=true ${SNAPCRAFT_PART_BUILD} ${SNAPCRAFT_PART_SRC}
ninja -C "${SNAPCRAFT_PART_BUILD}" -j4
meson test -C "${SNAPCRAFT_PART_BUILD}" --verbose
DESTDIR=${SNAPCRAFT_PART_INSTALL} ninja -C "${SNAPCRAFT_PART_BUILD}" install
override-prime: |
set -eux
snapcraftctl prime
for dir in usr/lib/*/gdk-pixbuf-2.0; do
if [ -d "${dir}" ]; then
for subdir in "${dir}"/*; do
if [ -d "${subdir}" ] && [ -d "${subdir}/loaders" ]; then
GDK_PIXBUF_MODULEDIR="$(pwd)/${subdir}/loaders" "${dir}/gdk-pixbuf-query-loaders" > "${subdir}/loaders.cache"
sed -i "s,$(pwd),," "${subdir}/loaders.cache"
fi
done
fi
done
update-mime-database -V usr/share/mime/
usr/lib/${SNAPCRAFT_ARCH_TRIPLET}/glib-2.0/glib-compile-schemas usr/share/glib-2.0/schemas
build-packages:
- curl
- meson
- docbook-xsl
- docbook-xml
- xsltproc
- ffmpeg
- libgdk-pixbuf2.0-dev
- libarchive-dev
- librsvg2-dev
- liblmdb-dev
- libglib2.0-dev
- libcurl4-gnutls-dev
- libcairo2-dev
- libfreetype6-dev
- libfontconfig1-dev
- libpango1.0-dev
stage-packages:
- gdb
- glib-networking
- ffmpeg
- optipng
- libc-bin
- libglu1-mesa
- freeglut3
- libslang2
- libarchive13
- libcairo2
- libcroco3
- libdatrie1
- libfontconfig1
- libfreetype6
- libgdk-pixbuf2.0-0
- libglib2.0-0
- libglib2.0-bin
- libgraphite2-3
- libharfbuzz0b
- libpango-1.0-0
- libpangocairo-1.0-0
- libpangoft2-1.0-0
- libpixman-1-0
- libpng16-16
- librsvg2-2
- librsvg2-common
- libcurl3-gnutls
- libthai0
- libx11-6
- libxau6
- libxcb-render0
- libxcb-shm0
- libxcb1
- libxdmcp6
- libxext6
- libxrender1
- locales
- locales-all
- shared-mime-info
after:
- appstream
- ldc
- gir-to-d
- glibd
appstream:
source: https://github.com/ximion/appstream.git
source-type: git
source-tag: v0.15.1
plugin: meson
meson-parameters:
- --prefix=/usr
- --buildtype=debugoptimized
- -Dcompose=true
- -Dapidocs=false
build-packages:
- build-essential
- docbook-xsl
- docbook-xml
- gettext
- gobject-introspection
- gperf
- itstool
- libgirepository1.0-dev
- libglib2.0-dev
- libxmlb-dev
- liblzma-dev
- libcurl4-gnutls-dev
- libstemmer-dev
- libxml2-dev
- libyaml-dev
- meson
- xsltproc
stage-packages:
- libicu66
- liblmdb0
- libcurl3-gnutls
- libstemmer0d
- libxml2
gir-to-d:
source: https://github.com/gtkd-developers/gir-to-d.git
source-type: git
source-tag: v0.22.0
plugin: meson
meson-parameters:
- --prefix=/usr
- --buildtype=release
build-packages:
- meson
after:
- ldc
glibd:
source: https://github.com/gtkd-developers/GlibD.git
source-type: git
source-tag: v2.3.0
plugin: meson
meson-parameters:
- --prefix=/usr
- --buildtype=release
build-packages:
- meson
- gobject-introspection
- libgirepository1.0-dev
- libglib2.0-dev
after:
- gir-to-d
- ldc
ldc:
plugin: dump
source: https://github.com/ldc-developers/ldc/releases/download/v1.28.0/ldc2-1.28.0-linux-x86_64.tar.xz
override-build: |
# link shared by default
sed -i '/\sswitches =/a \ "-link-defaultlib-shared",' etc/ldc2.conf
# remove unneeded libs
rm -rf lib32
snapcraftctl build
appstream-generator-0.8.6/src/ 0000775 0000000 0000000 00000000000 14160667064 0016306 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/src/app.d 0000664 0000000 0000000 00000016131 14160667064 0017235 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2021 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
import std.stdio;
import std.path : buildPath;
import std.file : getcwd;
import std.getopt;
import std.string : format;
import std.array : empty;
import core.stdc.stdlib : exit;
import asgen.logging;
import asgen.config;
import asgen.engine;
import asgen.defines : ASGEN_VERSION;
private immutable helpText =
"Usage:
appstream-generator [OPTION...] - AppStream Generator.
AppStream Metadata Generator
Subcommands:
run SUITE [SECTION] - Process new metadata for the given distribution suite and publish it.
cleanup - Cleanup old metadata and media files.
publish SUITE [SECTION] - Export all metadata and publish reports in the export directories.
remove-found SUITE - Drop all valid processed metadata and hints.
forget PKID - Drop all information we have about this (partial) package-id.
info PKID - Show information associated with this (full) package-id.
Help Options:
-h, --help Show help options
Application Options:
--version Show the program version.
--verbose Show extra debugging information.
--force Force action.
-w|--workspace Define the workspace location.
-c|--config Use the given configuration file.
--export-dir Override the workspace root export directory.";
version (unittest) {
void main () {}
} else {
private void createXdgRuntimeDir ()
{
// Ubuntu's Snappy package manager doesn't create the runtime
// data dir, and some of the libraries we depend on expect it
// to be available. Try to create the directory
import std.process : environment;
import std.array : empty;
import std.file : exists, mkdirRecurse, setAttributes;
import std.algorithm : startsWith;
import std.conv : octal;
immutable xdgRuntimeDir = environment.get("XDG_RUNTIME_DIR");
if (xdgRuntimeDir.empty || !xdgRuntimeDir.startsWith ("/"))
return; // nothing to do here
if (xdgRuntimeDir.exists)
return; // directory already exists
try {
mkdirRecurse (xdgRuntimeDir);
xdgRuntimeDir.setAttributes(octal!700);
} catch (Exception e) {
logDebug ("Unable to create XDG runtime dir: %s", e.msg);
return;
}
logDebug ("Created missing XDG runtime dir: %s", xdgRuntimeDir);
}
void main(string[] args)
{
string command;
bool verbose;
bool showHelp;
bool showVersion;
bool forceAction;
string wdir;
string exportDir;
string configFname;
// parse command-line options
try {
getopt (args,
"help|h", &showHelp,
"verbose", &verbose,
"version", &showVersion,
"force", &forceAction,
"workspace|w", &wdir,
"config|c", &configFname,
"export-dir", &exportDir);
} catch (Exception e) {
writeln ("Unable to parse parameters: ", e.msg);
exit (1);
}
if (showHelp) {
writeln (helpText);
return;
}
if (showVersion) {
writeln ("Generator version: ", ASGEN_VERSION);
return;
}
if (args.length < 2) {
writeln ("No subcommand specified!");
return;
}
// globally enable verbose mode, if requested
if (verbose)
asgen.logging.setVerbose (true);
auto conf = Config.get ();
if (configFname.empty) {
// if we don't have an explicit config file set, and also no
// workspace, take the current directory
if (wdir.empty)
wdir = getcwd ();
configFname = buildPath (wdir, "asgen-config.json");
}
try {
conf.loadFromFile (configFname, wdir, exportDir);
} catch (Exception e) {
writefln ("Unable to load configuration: %s", e.msg);
exit (4);
}
scope (exit) {
// ensure we clean up when the generator is done
import std.file : rmdirRecurse, exists;
if (conf.getTmpDir.exists)
rmdirRecurse (conf.getTmpDir ());
}
// ensure runtime dir exists, in case we are installed with Snappy
createXdgRuntimeDir ();
auto engine = new Engine ();
engine.forced = forceAction;
void ensureSuiteAndOrSectionParameterSet ()
{
if (args.length < 3) {
writeln ("Invalid number of parameters: You need to specify at least a suite name.");
exit (1);
}
if (args.length > 4) {
writeln ("Invalid number of parameters: You need to specify a suite name and (optionally) a section name.");
exit (1);
}
}
command = args[1];
switch (command) {
case "run":
case "process":
ensureSuiteAndOrSectionParameterSet ();
if (args.length == 3)
engine.run (args[2]);
else
engine.run (args[2], args[3]);
break;
case "process-file":
if (args.length < 5) {
writeln ("Invalid number of parameters: You need to specify a suite name, a section name and at least one file to process.");
exit (1);
}
engine.processFile (args[2], args[3], args[4..$]);
break;
case "publish":
ensureSuiteAndOrSectionParameterSet ();
if (args.length == 3)
engine.publish (args[2]);
else
engine.publish (args[2], args[3]);
break;
case "cleanup":
engine.runCleanup ();
break;
case "remove-found":
if (args.length != 3) {
writeln ("Invalid number of parameters: You need to specify a suite name.");
exit (1);
}
engine.removeHintsComponents (args[2]);
break;
case "forget":
if (args.length != 3) {
writeln ("Invalid number of parameters: You need to specify a package-id (partial IDs are allowed).");
exit (1);
}
engine.forgetPackage (args[2]);
break;
case "info":
if (args.length != 3) {
writeln ("Invalid number of parameters: You need to specify a package-id.");
exit (1);
}
engine.printPackageInfo (args[2]);
break;
default:
writeln (format ("The command '%s' is unknown.", command));
exit (1);
}
}
}
appstream-generator-0.8.6/src/asgen/ 0000775 0000000 0000000 00000000000 14160667064 0017403 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/src/asgen/backends/ 0000775 0000000 0000000 00000000000 14160667064 0021155 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/src/asgen/backends/alpinelinux/ 0000775 0000000 0000000 00000000000 14160667064 0023505 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/src/asgen/backends/alpinelinux/apkindexutils.d 0000664 0000000 0000000 00000010135 14160667064 0026536 0 ustar 00root root 0000000 0000000 module asgen.backends.alpinelinux.apkindexutils;
import std.algorithm : remove;
import std.algorithm.iteration : map;
import std.algorithm.searching : canFind;
import std.array : join, split;
import std.file : exists;
import std.format : format;
import std.path : buildPath;
import std.range : empty, InputRange, isForwardRange;
import std.string : splitLines, startsWith, strip;
import std.utf : validate;
import std.stdio;
import asgen.backends.alpinelinux.apkpkg;
import asgen.downloader : Downloader, DownloadException;
import asgen.logging : logDebug;
import asgen.utils : isRemote;
/**
* Struct representing a block inside of an APKINDEX. Each block, seperated by
* a newline, contains information about exactly one package.
*/
struct ApkIndexBlock {
string arch;
string maintainer;
string pkgname;
string pkgversion;
string pkgdesc;
@property string archiveName () {
return format ("%s-%s.apk", this.pkgname, this.pkgversion);
}
}
/**
* Range for looping over the contents of an APKINDEX, block by block.
*/
struct ApkIndexBlockRange {
this (string contents)
{
this.lines = contents.splitLines;
this.getNextBlock();
}
@property ApkIndexBlock front () const {
return this.currentBlock;
}
@property bool empty () {
return this.m_empty;
}
void popFront ()
{
this.getNextBlock ();
}
@property ApkIndexBlockRange save () { return this; }
private:
void getNextBlock () {
string[] completePair;
uint iterations = 0;
currentBlock = ApkIndexBlock();
foreach (currentLine; this.lines[this.lineDelta .. $]) {
iterations++;
if (currentLine == "") {
// next block for next package started
break;
} if (currentLine.canFind (":")) {
if (completePair.empty) {
completePair = [currentLine];
continue;
}
auto pair = completePair.join (" ").split (":");
this.setCurrentBlock (pair[0], pair[1]);
completePair = [currentLine];
} else {
completePair ~= currentLine.strip ();
}
}
this.lineDelta += iterations;
this.m_empty = this.lineDelta == this.lines.length;
}
void setCurrentBlock (string key, string value) {
switch (key) {
case "A":
this.currentBlock.arch = value;
break;
case "m":
this.currentBlock.maintainer = value;
break;
case "P":
this.currentBlock.pkgname = value;
break;
case "T":
this.currentBlock.pkgdesc = value;
break;
case "V":
this.currentBlock.pkgversion = value;
break;
default:
// We dont care about other keys
break;
}
}
string[] lines;
ApkIndexBlock currentBlock;
bool m_empty;
uint lineDelta;
}
static assert (isForwardRange!ApkIndexBlockRange);
/**
* Download apkindex if required. Returns the path to the local copy of the APKINDEX.
*/
immutable (string) downloadIfNecessary (const string prefix,
const string destPrefix,
const string srcFileName,
const string destFileName)
{
auto downloader = Downloader.get;
immutable filePath = buildPath (prefix, srcFileName);
immutable destFilePath = buildPath (destPrefix, destFileName);
if (filePath.isRemote) {
try {
downloader.downloadFile (filePath, destFilePath);
return destFilePath;
} catch (DownloadException e) {
logDebug ("Unable to download: %s", e.msg);
}
} else {
if (filePath.exists)
return filePath;
}
/* all extensions failed, so we failed */
throw new Exception ("Could not obtain any file matching %s".format (buildPath (prefix, srcFileName)));
}
appstream-generator-0.8.6/src/asgen/backends/alpinelinux/apkpkg.d 0000664 0000000 0000000 00000007653 14160667064 0025142 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2020 Rasmus Thomsen
*
* Based on the archlinux backend, which is:
* Copyright (C) 2016 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.alpinelinux.apkpkg;
import std.array : empty;
import std.format : format;
import std.path : baseName, buildNormalizedPath, buildPath;
import asgen.backends.interfaces;
import asgen.config : Config;
import asgen.downloader : Downloader;
import asgen.utils : isRemote;
import asgen.zarchive : ArchiveDecompressor;
final class AlpinePackage : Package
{
private:
string pkgname;
string pkgver;
string pkgarch;
string pkgmaintainer;
string[string] desc;
string pkgFname;
string localPkgFName;
string tmpDir;
string[] contentsL = null;
ArchiveDecompressor archive;
public:
this (string pkgname, string pkgver, string pkgarch)
{
this.pkgname = pkgname;
this.pkgver = pkgver;
this.pkgarch = pkgarch;
auto conf = Config.get ();
this.tmpDir = buildPath (conf.getTmpDir (), format ("%s-%s_%s", name, ver, arch));
}
override @property string name () const
{
return this.pkgname;
}
@property void name (string val)
{
this.pkgname = val;
}
override @property string ver () const
{
return this.pkgver;
}
@property void ver (string val)
{
this.pkgver = val;
}
override @property string arch () const
{
return this.pkgarch;
}
@property void arch(string val)
{
this.pkgarch = val;
}
override @property const(string[string]) description () const
{
return this.desc;
}
@property void filename (string fname)
{
this.pkgFname = fname;
}
override @property string getFilename ()
{
if (!this.localPkgFName.empty)
return this.localPkgFName;
if (pkgFname.isRemote) {
synchronized (this) {
auto dl = Downloader.get;
immutable path = buildNormalizedPath (this.tmpDir, this.pkgFname.baseName);
dl.downloadFile (this.pkgFname, path);
this.localPkgFName = path;
return this.localPkgFName;
}
} else {
this.localPkgFName = pkgFname;
return this.localPkgFName;
}
}
override @property string maintainer () const
{
return this.pkgmaintainer;
}
@property void maintainer (string maint)
{
this.pkgmaintainer = maint;
}
void setDescription (string text, string locale)
{
this.desc[locale] = text;
}
override const(ubyte)[] getFileData (string fname)
{
if (!this.archive.isOpen())
this.archive.open (this.getFilename);
return this.archive.readData (fname);
}
@property override string[] contents ()
{
if (!this.contentsL.empty)
return this.contentsL;
ArchiveDecompressor ad;
ad.open (this.getFilename);
this.contentsL = ad.readContents ();
return this.contentsL;
}
@property void contents (string[] c)
{
this.contentsL = c;
}
override void finish ()
{
}
}
appstream-generator-0.8.6/src/asgen/backends/alpinelinux/apkpkgindex.d 0000664 0000000 0000000 00000010534 14160667064 0026162 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2020 Rasmus Thomsen
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.alpinelinux.apkpkgindex;
import std.array : appender;
import std.conv : to;
import std.exception : enforce;
import std.file : exists;
import std.format : format;
import std.path : baseName, buildPath;
import std.utf : validate;
import asgen.config : Config;
import asgen.logging : logError;
import asgen.zarchive : ArchiveDecompressor;
import asgen.utils : escapeXml, isRemote;
import asgen.backends.interfaces;
import asgen.backends.alpinelinux.apkpkg;
import asgen.backends.alpinelinux.apkindexutils;
final class AlpinePackageIndex : PackageIndex
{
private:
string rootDir;
string tmpDir;
Package[][string] pkgCache;
public:
this (string dir)
{
if (!dir.isRemote)
enforce (exists (dir), format ("Directory '%s' does not exist.", dir));
this.rootDir = dir;
auto conf = Config.get ();
tmpDir = buildPath (conf.getTmpDir, dir.baseName);
}
override void release ()
{
pkgCache = null;
}
private void setPkgDescription (AlpinePackage pkg, string pkgDesc)
{
if (pkgDesc is null)
return;
auto desc = "
%s
".format (pkgDesc.escapeXml);
pkg.setDescription (desc, "C");
}
private Package[] loadPackages (string suite, string section, string arch)
{
auto apkRootPath = buildPath (rootDir, suite, section, arch);
auto indexFPath = downloadIfNecessary(apkRootPath, tmpDir, "APKINDEX.tar.gz", format("APKINDEX-%s-%s-%s.tar.gz", suite, section, arch));
AlpinePackage[string] pkgsMap;
ArchiveDecompressor ad;
ad.open (indexFPath);
auto indexString = cast(string) ad.readData ("APKINDEX");
validate (indexString);
auto range = ApkIndexBlockRange (indexString);
foreach (pkgInfo; range) {
auto fileName = pkgInfo.archiveName;
AlpinePackage pkg;
if (fileName in pkgsMap) {
pkg = pkgsMap[fileName];
} else {
pkg = new AlpinePackage (pkgInfo.pkgname, pkgInfo.pkgversion, pkgInfo.arch);
pkgsMap[fileName] = pkg;
}
pkg.filename = buildPath(rootDir, suite, section, arch, fileName);
pkg.maintainer = pkgInfo.maintainer;
setPkgDescription(pkg, pkgInfo.pkgdesc);
}
// perform a sanity check, so we will never emit invalid packages
auto pkgs = appender!(Package[]);
if (pkgsMap.length > 20)
pkgs.reserve ((pkgsMap.length.to!long - 10).to!size_t);
foreach (ref pkg; pkgsMap.byValue ())
if (pkg.isValid)
pkgs ~= pkg;
else
logError ("Found an invalid package (name, architecture or version is missing). This is a bug.");
return pkgs.data;
}
override Package[] packagesFor (string suite, string section, string arch, bool withLongDescs = true)
{
immutable id = "%s-%s-%s".format (suite, section, arch);
if (id !in pkgCache) {
auto pkgs = loadPackages (suite, section, arch);
synchronized (this)
pkgCache[id] = pkgs;
}
return pkgCache[id];
}
Package packageForFile (string fname, string suite = null, string section = null)
{
// Alpine currently doesn't have a way other than querying a web API
// to tell what package owns a file, unless that packages is installed
// on the system.
return null;
}
bool hasChanges (DataStore dstore, string suite, string section, string arch)
{
return true;
}
}
appstream-generator-0.8.6/src/asgen/backends/alpinelinux/package.d 0000664 0000000 0000000 00000001657 14160667064 0025256 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2020 Rasmus Thomsen
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.alpinelinux;
public import asgen.backends.alpinelinux.apkpkg;
public import asgen.backends.alpinelinux.apkpkgindex;
appstream-generator-0.8.6/src/asgen/backends/archlinux/ 0000775 0000000 0000000 00000000000 14160667064 0023152 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/src/asgen/backends/archlinux/alpkg.d 0000664 0000000 0000000 00000004720 14160667064 0024420 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.archlinux.alpkg;
import std.stdio;
import std.string;
import std.array : empty;
import asgen.logging;
import asgen.zarchive;
import asgen.backends.interfaces;
final class ArchPackage : Package
{
private:
string pkgname;
string pkgver;
string pkgarch;
string pkgmaintainer;
string[string] desc;
string pkgFname;
string[] contentsL;
ArchiveDecompressor archive;
public:
override
@property string name () const { return pkgname; }
@property void name (string val) { pkgname = val; }
override
@property string ver () const { return pkgver; }
@property void ver (string val) { pkgver = val; }
override
@property string arch () const { return pkgarch; }
@property void arch (string val) { pkgarch = val; }
override
@property const(string[string]) description () const { return desc; }
@property void filename (string fname) { pkgFname = fname; }
override
@property string getFilename () const { return pkgFname; }
override
@property string maintainer () const { return pkgmaintainer; }
@property void maintainer (string maint) { pkgmaintainer = maint; }
void setDescription (string text, string locale)
{
desc[locale] = text;
}
override
const(ubyte)[] getFileData (string fname)
{
if (!archive.isOpen ()) {
archive.open (this.getFilename);
}
return archive.readData (fname);
}
@property override
string[] contents ()
{
return contentsL;
}
@property
void contents (string[] c)
{
contentsL = c;
}
override
void finish ()
{
}
}
appstream-generator-0.8.6/src/asgen/backends/archlinux/alpkgindex.d 0000664 0000000 0000000 00000012256 14160667064 0025453 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.archlinux.alpkgindex;
import std.stdio;
import std.path;
import std.string;
import std.algorithm : canFind;
import std.array : appender;
import std.conv : to;
static import std.file;
import asgen.logging;
import asgen.zarchive;
import asgen.utils : escapeXml;
import asgen.backends.interfaces;
import asgen.backends.archlinux.alpkg;
import asgen.backends.archlinux.listfile;
final class ArchPackageIndex : PackageIndex
{
private:
string rootDir;
Package[][string] pkgCache;
public:
this (string dir)
{
this.rootDir = dir;
if (!std.file.exists (dir))
throw new Exception ("Directory '%s' does not exist.", dir);
}
void release ()
{
pkgCache = null;
}
private void setPkgDescription (ArchPackage pkg, string pkgDesc)
{
if (pkgDesc is null)
return;
auto desc = "
%s
".format (pkgDesc.escapeXml);
pkg.setDescription (desc, "C");
}
private Package[] loadPackages (string suite, string section, string arch)
{
auto pkgRoot = buildPath (rootDir, suite, section, "os", arch);
auto listsTarFname = buildPath (pkgRoot, format ("%s.files.tar.gz", section));
if (!std.file.exists (listsTarFname)) {
logWarning ("Package lists tarball '%s' does not exist.", listsTarFname);
return [];
}
ArchiveDecompressor ad;
ad.open (listsTarFname);
logDebug ("Opened: %s", listsTarFname);
ArchPackage[string] pkgsMap;
foreach (entry; ad.read ()) {
auto archPkid = dirName (entry.fname);
ArchPackage pkg;
if (archPkid in pkgsMap) {
pkg = pkgsMap[archPkid];
} else {
pkg = new ArchPackage ();
pkgsMap[archPkid] = pkg;
}
auto infoBaseName = baseName (entry.fname);
if (infoBaseName == "desc") {
// we have the description file, add information to this package
auto descF = new ListFile ();
descF.loadData (entry.data);
pkg.name = descF.getEntry ("NAME");
pkg.ver = descF.getEntry ("VERSION");
pkg.arch = descF.getEntry ("ARCH");
pkg.maintainer = descF.getEntry ("PACKAGER");
pkg.filename = buildPath (pkgRoot, descF.getEntry ("FILENAME"));
setPkgDescription (pkg, descF.getEntry ("DESC"));
} else if (infoBaseName == "files") {
// we found a content index, add content information to the package
auto filesF = new ListFile ();
filesF.loadData (entry.data);
auto filesStr = filesF.getEntry ("FILES");
if (filesStr is null) {
if (!pkg.name.canFind ("-meta")) {
logWarning ("Package '%s' has no file list set. Ignoring it.", pkg.toString ());
continue;
}
}
auto contents = appender!(string[]);
foreach (l; filesStr.splitLines ())
contents ~= "/" ~ l;
pkg.contents = contents.data;
}
}
// perform a sanity check, so we will never emit invalid packages
auto pkgs = appender!(Package[]);
if (pkgsMap.length > 20)
pkgs.reserve ((pkgsMap.length.to!long - 10).to!size_t);
foreach (ref pkg; pkgsMap.byValue ()) {
if (pkg.isValid)
pkgs ~= pkg;
else
logError ("Found an invalid package (name, architecture or version is missing). This is a bug.");
}
return pkgs.data;
}
Package[] packagesFor (string suite, string section, string arch, bool withLongDescs = true)
{
if ((suite == "arch") || (suite == "archlinux"))
suite = "";
immutable id = "%s-%s-%s".format (suite, section, arch);
if (id !in pkgCache) {
auto pkgs = loadPackages (suite, section, arch);
synchronized (this) pkgCache[id] = pkgs;
}
return pkgCache[id];
}
Package packageForFile (string fname, string suite = null, string section = null)
{
return null; // FIXME: not implemented
}
bool hasChanges (DataStore dstore, string suite, string section, string arch)
{
return true;
}
}
appstream-generator-0.8.6/src/asgen/backends/archlinux/listfile.d 0000664 0000000 0000000 00000005056 14160667064 0025140 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.archlinux.listfile;
import std.stdio;
import std.string;
final class ListFile
{
private:
string[string] entries;
public:
this ()
{
}
void loadData (const(ubyte)[] data)
{
string[] content;
auto dataStr = cast(string) data;
content = dataStr.splitLines ();
string blockName = null;
foreach (l; content) {
if ((l.startsWith ("%")) && (l.endsWith ("%"))) {
blockName = l[1..$-1];
continue;
}
if (l == "") {
blockName = null;
continue;
}
if (blockName !is null) {
if (blockName in entries)
entries[blockName] ~= "\n" ~ l;
else
entries[blockName] = l;
continue;
}
}
}
string getEntry (string name)
{
auto resP = name in entries;
if (resP is null)
// we found nothing
return null;
return *resP;
}
}
unittest
{
writeln ("TEST: ", "Backend::Archlinux - ListFile");
string data = "%FILENAME%
a2ps-4.14-6-x86_64.pkg.tar.xz
%NAME%
a2ps
%VERSION%
4.14-6
%DESC%
An Any to PostScript filter
%CSIZE%
629320
%MULTILINE%
Blah1
BLUBB2
EtcEtcEtc3
%SHA256SUM%
a629a0e0eca0d96a97eb3564f01be495772439df6350600c93120f5ac7f3a1b5";
auto lf = new ListFile ();
lf.loadData (cast(ubyte[]) data);
assert (lf.getEntry ("FILENAME") == "a2ps-4.14-6-x86_64.pkg.tar.xz");
assert (lf.getEntry ("VERSION") == "4.14-6");
assert (lf.getEntry ("MULTILINE") == "Blah1\nBLUBB2\nEtcEtcEtc3");
assert (lf.getEntry ("SHA256SUM") == "a629a0e0eca0d96a97eb3564f01be495772439df6350600c93120f5ac7f3a1b5");
}
appstream-generator-0.8.6/src/asgen/backends/archlinux/package.d 0000664 0000000 0000000 00000000204 14160667064 0024706 0 ustar 00root root 0000000 0000000
module asgen.backends.archlinux;
public import asgen.backends.archlinux.alpkg;
public import asgen.backends.archlinux.alpkgindex;
appstream-generator-0.8.6/src/asgen/backends/debian/ 0000775 0000000 0000000 00000000000 14160667064 0022377 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/src/asgen/backends/debian/debpkg.d 0000664 0000000 0000000 00000025633 14160667064 0024011 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2017 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.debian.debpkg;
import std.stdio : File, writeln;
import std.string : format, strip, endsWith, splitLines, join, split;
import std.path : buildNormalizedPath, buildPath, baseName;
import std.array : empty, appender;
import std.file : rmdirRecurse, mkdirRecurse;
import std.typecons : Nullable;
static import std.file;
import asgen.config;
import asgen.logging;
import asgen.zarchive;
import asgen.backends.interfaces;
import asgen.downloader : Downloader;
import asgen.utils : isRemote;
/**
* Helper class for simple deduplication of package descriptions
* between packages of different architectures in memory.
*/
final class DebPackageLocaleTexts
{
string[string] summary; /// map of localized package short summaries
string[string] description; /// map of localized package descriptions
void setDescription (string text, string locale)
{
synchronized (this)
description[locale] = text;
}
void setSummary (string text, string locale)
{
synchronized (this)
summary[locale] = text;
}
}
/**
* Representation of a Debian binary package
*/
class DebPackage : Package
{
private:
string pkgname;
string pkgver;
string pkgarch;
string pkgmaintainer;
DebPackageLocaleTexts descTexts;
Nullable!GStreamer gstreamer;
bool contentsRead;
string[] contentsL;
string tmpDir;
ArchiveDecompressor controlArchive;
ArchiveDecompressor dataArchive;
string debFname;
string localDebFname;
public:
final @property override string name () const { return pkgname; }
final @property override string ver () const { return pkgver; }
final @property override string arch () const { return pkgarch; }
final @property void name (string s) { pkgname = s; }
final @property void ver (string s) { pkgver = s; }
final @property void arch (string s) { pkgarch = s; }
final @property override Nullable!GStreamer gst () { return gstreamer; }
final @property void gst (GStreamer gst) { gstreamer = gst; }
final @property override const(string[string]) description () const { return descTexts.description; }
final @property override const(string[string]) summary () const { return descTexts.summary; }
final @property void filename (string fname) { debFname = fname; localDebFname = null; }
override final
string getFilename () {
if (!localDebFname.empty)
return localDebFname;
if (debFname.isRemote) {
synchronized (this) {
auto dl = Downloader.get;
immutable path = buildNormalizedPath (tmpDir, debFname.baseName);
dl.downloadFile (debFname, path);
localDebFname = path;
return localDebFname;
}
} else {
localDebFname = debFname;
return debFname;
}
}
override
final @property string maintainer () const { return pkgmaintainer; }
final @property void maintainer (string maint) { pkgmaintainer = maint; }
this (string pname, string pver, string parch, DebPackageLocaleTexts l10nTexts = null)
{
pkgname = pname;
pkgver = pver;
pkgarch = parch;
descTexts = l10nTexts;
if (descTexts is null)
descTexts = new DebPackageLocaleTexts;
contentsRead = false;
updateTmpDirPath ();
}
~this ()
{
// FIXME: We can't properly clean up because we can't GC-allocate in a destructor (leads to crashes),
// see if this is fixed in a future version of D, or simply don't use the GC in close ().
// close ();
}
final void updateTmpDirPath ()
{
auto conf = Config.get ();
tmpDir = buildPath (conf.getTmpDir (), format ("%s-%s_%s", name, ver, arch));
}
final void setDescription (string text, string locale)
{
descTexts.setDescription (text, locale);
}
final void setSummary (string text, string locale)
{
descTexts.setSummary (text, locale);
}
final setLocalizedTexts (DebPackageLocaleTexts l10nTexts)
{
assert (l10nTexts !is null);
descTexts = l10nTexts;
}
@property
final DebPackageLocaleTexts localizedTexts ()
{
return descTexts;
}
private auto openPayloadArchive ()
{
import std.regex : ctRegex;
synchronized(this) {
if (dataArchive.isOpen)
return dataArchive;
ArchiveDecompressor ad;
// extract the payload to a temporary location first
ad.open (this.getFilename);
mkdirRecurse (tmpDir);
auto files = ad.extractFilesByRegex (ctRegex!(r"data\.*"), tmpDir);
if (files.length == 0)
throw new Exception ("Unable to find the payload tarball in Debian package: %s".format (this.getFilename));
immutable dataArchiveFname = files[0];
dataArchive.open (dataArchiveFname);
return dataArchive;
}
}
final void extractPackage (const string dest = buildPath (tmpDir, name))
{
import std.file : exists;
import std.regex : ctRegex;
synchronized(this) {
if (!dest.exists)
mkdirRecurse (dest);
auto pa = openPayloadArchive ();
pa.extractArchive (dest);
}
}
private final auto openControlArchive ()
{
import std.regex : ctRegex;
synchronized(this) {
if (controlArchive.isOpen)
return controlArchive;
ArchiveDecompressor ad;
// extract the payload to a temporary location first
ad.open (this.getFilename);
mkdirRecurse (tmpDir);
auto files = ad.extractFilesByRegex (ctRegex!(r"control\.*"), tmpDir);
if (files.empty)
throw new Exception ("Unable to find control data in Debian package: %s".format (this.getFilename));
immutable controlArchiveFname = files[0];
controlArchive.open (controlArchiveFname);
return controlArchive;
}
}
override final
const(ubyte)[] getFileData (string fname)
{
auto pa = openPayloadArchive ();
return pa.readData (fname);
}
@property override final
string[] contents ()
{
import std.utf : toUTF8;
if (contentsRead)
return contentsL;
if (pkgname.endsWith ("icon-theme")) {
// the md5sums file does not contain symbolic links - while that is okay-ish for regular
// packages, it is not acceptable for icon themes, since those rely on symlinks to provide
// aliases for certain icons. So, use the slow method for reading contents information here.
auto pa = openPayloadArchive ();
contentsL = pa.readContents ();
contentsRead = true;
return contentsL;
}
// use the md5sums file of the .deb control archive to determine
// the contents of this package.
// this is way faster than going through the payload directly, and
// has the same accuracy.
auto ca = openControlArchive ();
const(ubyte)[] md5sumsData;
try {
md5sumsData = ca.readData ("./md5sums");
} catch (Exception e) {
logWarning ("Could not read md5sums file for package %s: %s", this.id, e.msg);
return [];
}
auto md5sums = cast(string) md5sumsData;
try {
md5sums = md5sums.toUTF8;
} catch (Exception e) {
logError ("Could not decode md5sums file for package %s: %s", this.id, e.msg);
return [];
}
auto contentsAppender = appender!(string[]);
contentsAppender.reserve (20);
foreach (line; md5sums.splitLines ()) {
auto parts = line.split (" ");
if (parts.length <= 0)
continue;
string c = join (parts[1..$], " ");
contentsAppender.put ("/" ~ c);
}
contentsL = contentsAppender.data;
contentsRead = true;
return contentsL;
}
/**
* Get the "control" file information from the control archive
* in the Debian package.
* This is useful to get information from a package directly, e.g.
* for processing a single package.
*/
final auto readControlInformation ()
{
import std.utf : toUTF8;
import asgen.backends.debian.tagfile : TagFile;
auto ca = openControlArchive ();
const(ubyte)[] controlData;
try {
controlData = ca.readData ("./control");
} catch (Exception e) {
logError ("Could not read control file for package %s: %s", this.id, e.msg);
return null;
}
auto controlStr = cast(string) controlData;
try {
controlStr = controlStr.toUTF8;
} catch (Exception e) {
logError ("Could not decode control file for package %s: %s", this.id, e.msg);
return null;
}
auto tf = new TagFile ();
tf.load (controlStr);
return tf;
}
override final
void cleanupTemp ()
{
synchronized(this) {
if (controlArchive.isOpen)
controlArchive.close ();
if (dataArchive.isOpen)
dataArchive.close ();
try {
if (std.file.exists (tmpDir)) {
/* Whenever we delete the temporary directory, we need to
* forget about the local file too, since (if it's remote) that
* was downloaded into there. */
logDebug ("Deleting temporary directory %s", tmpDir);
localDebFname = null;
rmdirRecurse (tmpDir);
}
} catch (Exception e) {
// we ignore any error
logDebug ("Unable to remove temporary directory: %s (%s)", tmpDir, e.msg);
}
}
}
override final
void finish ()
{
cleanupTemp ();
}
}
appstream-generator-0.8.6/src/asgen/backends/debian/debpkgindex.d 0000664 0000000 0000000 00000032315 14160667064 0025034 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2018 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.debian.debpkgindex;
import std.stdio;
import std.path;
import std.string;
import std.algorithm : remove;
import std.array : appender, array;
import std.conv : to;
import std.typecons : scoped;
static import std.file;
import asgen.logging;
import asgen.backends.interfaces;
import asgen.backends.debian.tagfile;
import asgen.backends.debian.debpkg;
import asgen.backends.debian.debutils;
import asgen.config;
import asgen.utils : escapeXml, getTextFileContents, isRemote;
class DebianPackageIndex : PackageIndex
{
private:
string rootDir;
Package[][string] pkgCache;
// index of localized text for a specific package name
DebPackageLocaleTexts[string] l10nTextIndex;
bool[string] indexChanged;
protected:
string tmpDir;
public:
this (string dir)
{
pkgCache.clear ();
this.rootDir = dir;
if (!dir.isRemote && !std.file.exists (dir))
throw new Exception ("Directory '%s' does not exist.".format (dir));
auto conf = Config.get ();
tmpDir = buildPath (conf.getTmpDir, dir.baseName);
}
void release ()
{
pkgCache.clear ();
l10nTextIndex.clear ();
indexChanged = null;
}
private immutable(string[]) findTranslations (const string suite, const string section)
{
import std.regex : matchFirst, regex;
immutable inRelease = buildPath (rootDir, "dists", suite, "InRelease");
auto translationregex = r"%s/i18n/Translation-(\w+)$".format (section).regex;
bool[string] ret;
try {
synchronized (this) {
const inReleaseContents = getTextFileContents (inRelease);
foreach (const ref entry; inReleaseContents) {
auto match = entry.matchFirst (translationregex);
if (match.empty)
continue;
ret[match[1]] = true;
}
}
} catch (Exception ex) {
logWarning ("Could not get %s, will assume 'en' is available.", inRelease);
return ["en"];
}
return cast(immutable) array (ret.byKey);
}
/**
* Convert a Debian package description to a description
* that looks nice-ish in AppStream clients.
*/
private string packageDescToAppStreamDesc (string[] lines)
{
// TODO: We actually need a Markdown-ish parser here if we want
// to support listings in package descriptions properly.
auto description = appender!string;
description.reserve (80);
description ~= "
";
bool first = true;
foreach (l; lines) {
if (l.strip () == ".") {
description ~= "
\n
";
first = true;
continue;
}
if (first)
first = false;
else
description ~= " ";
description ~= escapeXml (l);
}
description ~= "
";
return description.data;
}
private void loadPackageLongDescs (ref DebPackage[string] pkgs, string suite, string section)
{
immutable langs = findTranslations (suite, section);
logDebug ("Found translations for: %s", langs.join(", "));
foreach (const ref lang; langs) {
string fname;
immutable fullPath = buildPath ("dists",
suite,
section,
"i18n",
/* here we explicitly substitute a
* "%s", because
* downloadIfNecessary will put the
* file extension there */
"Translation-%s.%s".format(lang, "%s"));
try {
synchronized (this) {
fname = downloadIfNecessary (rootDir, tmpDir, fullPath);
}
} catch (Exception ex) {
logDebug ("No translations for %s in %s/%s", lang, suite, section);
continue;
}
auto tagf = scoped!TagFile ();
tagf.open (fname);
do {
immutable pkgname = tagf.readField ("Package");
immutable rawDesc = tagf.readField ("Description-%s".format (lang));
if (!pkgname)
continue;
if (!rawDesc)
continue;
auto pkg = pkgs.get (pkgname, null);
if (pkg is null)
continue;
immutable textPkgId = "%s/%s".format (pkg.name, pkg.ver);
DebPackageLocaleTexts l10nTexts;
synchronized (this)
l10nTexts = l10nTextIndex.get (textPkgId, null);
if (l10nTexts !is null) {
// we already fetched this information
pkg.setLocalizedTexts (l10nTexts);
}
// read new localizations
l10nTexts = pkg.localizedTexts;
synchronized (this)
l10nTextIndex[textPkgId] = l10nTexts;
auto split = rawDesc.split ("\n");
if (split.length < 2)
continue;
if (lang == "en")
l10nTexts.setSummary (split[0], "C");
l10nTexts.setSummary (split[0], lang);
// NOTE: .remove() removes the element, but does not alter the
// length of the array. Bug? (this is why we slice the array
// here)
split = split[1..$];
immutable description = packageDescToAppStreamDesc (split);
if (lang == "en")
l10nTexts.setDescription (description, "C");
l10nTexts.setDescription (description, lang);
pkg.setLocalizedTexts (l10nTexts);
} while (tagf.nextSection ());
}
}
private string getIndexFile (string suite, string section, string arch)
{
immutable path = buildPath ("dists", suite, section, "binary-%s".format (arch));
synchronized (this) {
return downloadIfNecessary (rootDir, tmpDir, buildPath (path, "Packages.%s"));
}
}
protected DebPackage newPackage (string name, string ver, string arch)
{
return new DebPackage (name, ver, arch);
}
private DebPackage[] loadPackages (string suite, string section, string arch, bool withLongDescs = true)
{
auto indexFname = getIndexFile (suite, section, arch);
if (!std.file.exists (indexFname)) {
logWarning ("Archive package index file '%s' does not exist.", indexFname);
return [];
}
auto tagf = scoped!TagFile ();
tagf.open (indexFname);
logDebug ("Opened: %s", indexFname);
DebPackage[string] pkgs;
do {
import std.algorithm : map;
import std.array : array;
auto name = tagf.readField ("Package");
auto ver = tagf.readField ("Version");
auto fname = tagf.readField ("Filename");
auto pkgArch = tagf.readField ("Architecture");
if (!name)
continue;
// sanity check: We only allow arch:all mixed in with packages from other architectures
if (pkgArch != "all")
pkgArch = arch;
auto pkg = newPackage (name, ver, pkgArch);
pkg.filename = buildPath (rootDir, fname);
pkg.maintainer = tagf.readField ("Maintainer");
immutable decoders = tagf.readField("Gstreamer-Decoders")
.split(";")
.map!strip.array;
immutable encoders = tagf.readField("Gstreamer-Encoders")
.split(";")
.map!strip.array;
immutable elements = tagf.readField("Gstreamer-Elements")
.split(";")
.map!strip.array;
immutable uri_sinks = tagf.readField("Gstreamer-Uri-Sinks")
.split(";")
.map!strip.array;
immutable uri_sources = tagf.readField("Gstreamer-Uri-Sources")
.split(";")
.map!strip.array;
auto gst = new GStreamer (decoders, encoders, elements, uri_sinks, uri_sources);
if (gst.isNotEmpty)
pkg.gst = gst;
if (!pkg.isValid ()) {
logWarning ("Found invalid package (%s)! Skipping it.", pkg.toString ());
continue;
}
// filter out the most recent package version in the packages list
auto epkg = pkgs.get (name, null);
if (epkg !is null) {
if (compareVersions (epkg.ver, pkg.ver) > 0)
continue;
}
pkgs[name] = pkg;
} while (tagf.nextSection ());
// load long descriptions
if (withLongDescs)
loadPackageLongDescs (pkgs, suite, section);
return array(pkgs.byValue);
}
Package[] packagesFor (string suite, string section, string arch, bool withLongDescs = true)
{
immutable id = "%s/%s/%s".format (suite, section, arch);
if (id !in pkgCache) {
auto pkgs = loadPackages (suite, section, arch, withLongDescs);
synchronized (this) pkgCache[id] = to!(Package[]) (pkgs);
}
return pkgCache[id];
}
Package packageForFile (string fname, string suite = null, string section = null)
{
auto pkg = newPackage ("", "", "");
pkg.filename = fname;
auto tf = pkg.readControlInformation ();
if (tf is null)
throw new Exception ("Unable to read control information for package %s".format (fname));
pkg.name = tf.readField ("Package");
pkg.ver = tf.readField ("Version");
pkg.arch = tf.readField ("Architecture");
if (pkg.name is null || pkg.ver is null || pkg.arch is null)
throw new Exception ("Unable to get control data for package %s".format (fname));
immutable rawDesc = tf.readField ("Description");
auto dSplit = rawDesc.split ("\n");
if (dSplit.length >= 2) {
pkg.setSummary (dSplit[0], "C");
dSplit = dSplit[1..$];
immutable description = packageDescToAppStreamDesc (dSplit);
pkg.setDescription (description, "C");
}
// ensure we have a meaningful temporary directory name
pkg.updateTmpDirPath ();
return pkg.to!Package;
}
final bool hasChanges (DataStore dstore, string suite, string section, string arch)
{
import std.json;
auto indexFname = getIndexFile (suite, section, arch);
// if the file doesn't exit, we will emit a warning later anyway, so we just ignore this here
if (!std.file.exists (indexFname))
return true;
// check our cache on whether the index had changed
if (indexFname in indexChanged)
return indexChanged[indexFname];
std.datetime.SysTime mtime;
std.datetime.SysTime atime;
std.file.getTimes (indexFname, atime, mtime);
auto currentTime = mtime.toUnixTime ();
auto repoInfo = dstore.getRepoInfo (suite, section, arch);
scope (exit) {
repoInfo.object["mtime"] = JSONValue (currentTime);
dstore.setRepoInfo (suite, section, arch, repoInfo);
}
if ("mtime" !in repoInfo.object) {
indexChanged[indexFname] = true;
return true;
}
auto pastTime = repoInfo["mtime"].integer;
if (pastTime != currentTime) {
indexChanged[indexFname] = true;
return true;
}
indexChanged[indexFname] = false;
return false;
}
}
unittest {
import std.algorithm.sorting : sort;
import asgen.utils : getTestSamplesDir;
writeln ("TEST: ", "DebianPackageIndex");
auto pi = new DebianPackageIndex (buildPath (getTestSamplesDir (), "debian"));
assert (sort(pi.findTranslations ("sid", "main").dup) ==
sort(["en", "ca", "cs", "da", "de", "de_DE", "el", "eo", "es",
"eu", "fi", "fr", "hr", "hu", "id", "it", "ja", "km", "ko",
"ml", "nb", "nl", "pl", "pt", "pt_BR", "ro", "ru", "sk",
"sr", "sv", "tr", "uk", "vi", "zh", "zh_CN", "zh_TW"]));
}
appstream-generator-0.8.6/src/asgen/backends/debian/debutils.d 0000664 0000000 0000000 00000017556 14160667064 0024375 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016 Matthias Klumpp
* Copyright (C) The APT development team.
* Copyright (C) 2016 Canonical Ltd
* Author(s): Iain Lane
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.debian.debutils;
import std.string;
static import std.file;
import asgen.logging;
import asgen.downloader : Downloader, DownloadException;
import asgen.utils : isRemote;
/**
* If prefix is remote, download the first of (prefix + suffix).{xz,bz2,gz},
* otherwise check if any of (prefix + suffix).{xz,bz2,gz} exists.
*
* Returns: Path to the file, which is guaranteed to exist.
*
* Params:
* prefix = First part of the address, i.e.
* "http://ftp.debian.org/debian/" or "/srv/mirrors/debian/"
* destPrefix = If the file is remote, the directory to save it under,
* which is created if necessary.
* suffix = the rest of the address, so that (prefix +
* suffix).format({xz,bz2,gz}) is a full path or URL, i.e.
* "dists/unstable/main/binary-i386/Packages.%s". The suffix must
* contain exactly one "%s"; this function is only suitable for
* finding `.xz`, `.bz2` and `.gz` files.
*/
immutable (string) downloadIfNecessary (const string prefix,
const string destPrefix,
const string suffix,
Downloader downloader = null)
{
import std.path : buildPath;
if (downloader is null)
downloader = Downloader.get;
immutable exts = ["xz", "bz2", "gz"];
foreach (ref ext; exts) {
immutable fileName = format (buildPath (prefix, suffix), ext);
immutable destFileName = format (buildPath (destPrefix, suffix), ext);
if (fileName.isRemote) {
try {
downloader.downloadFile (fileName, destFileName);
return destFileName;
} catch (DownloadException e) {
logDebug ("Unable to download: %s", e.msg);
}
} else {
if (std.file.exists (fileName))
return fileName;
}
}
/* all extensions failed, so we failed */
throw new Exception ("Could not obtain any file matching %s".format (buildPath (prefix, suffix)));
}
/**
* This compares a fragment of the version. This is a slightly adapted
* version of what dpkg uses in dpkg/lib/dpkg/version.c.
* In particular, the a | b = NULL check is removed as we check this in the
* caller, we use an explicit end for a | b strings and we check ~ explicit.
*/
private int order (char c) pure
{
import std.ascii : isAlpha, isDigit;
if (c.isDigit)
return 0;
else if (c.isAlpha)
return c;
else if (c == '~')
return -1;
else if (c)
return c + 256;
else
return 0;
}
/**
* Iterate over the whole string
* What this does is to split the whole string into groups of
* numeric and non numeric portions. For instance:
* a67bhgs89
* Has 4 portions 'a', '67', 'bhgs', '89'. A more normal:
* 2.7.2-linux-1
* Has '2', '.', '7', '.' ,'-linux-','1'
*/
private int cmpFragment (const(immutable(char)*) a, const(immutable(char)*) aEnd,
const(immutable(char)*) b, const(immutable(char)*) bEnd) @trusted pure
{
import std.ascii;
immutable(char) *lhs = a;
immutable(char) *rhs = b;
while (lhs != aEnd && rhs != bEnd) {
int first_diff = 0;
while (lhs != aEnd && rhs != bEnd && (!(*lhs).isDigit || !(*rhs).isDigit)) {
int vc = order (*lhs);
int rc = order (*rhs);
if (vc != rc)
return vc - rc;
++lhs; ++rhs;
}
while (*lhs == '0')
++lhs;
while (*rhs == '0')
++rhs;
while ((*lhs).isDigit && (*rhs).isDigit) {
if (!first_diff)
first_diff = *lhs - *rhs;
++lhs;
++rhs;
}
if ((*lhs).isDigit)
return 1;
if ((*rhs).isDigit)
return -1;
if (first_diff)
return first_diff;
}
// The strings must be equal
if (lhs == aEnd && rhs == bEnd)
return 0;
// lhs is shorter
if (lhs == aEnd) {
if (*rhs == '~')
return 1;
return -1;
}
// rhs is shorter
if (rhs == bEnd) {
if (*lhs == '~')
return -1;
return 1;
}
// Shouldn't happen
return 1;
}
// import from string.h, needs glibc
private extern(C) void *memrchr (const void *s, int c, size_t n) @system pure;
/**
* Compare two Debian-style version numbers.
*/
int compareVersions (const string a, const string b) @trusted pure
{
import core.stdc.string;
immutable(char) *ac = a.toStringz;
immutable(char) *bc = b.toStringz;
immutable(char*) aEnd = ac + a.length;
immutable(char*) bEnd = bc + b.length;
// Strip off the epoch and compare it
auto lhs = cast(immutable(char)*) memchr (ac, ':', aEnd - ac);
auto rhs = cast(immutable(char)*) memchr (bc, ':', bEnd - bc);
if (lhs is null)
lhs = ac;
if (rhs is null)
rhs = bc;
// Special case: a zero epoch is the same as no epoch,
// so remove it.
if (lhs != ac) {
for (; *ac == '0'; ++ac) {}
if (ac == lhs) {
++ac;
++lhs;
}
}
if (rhs != bc) {
for (; *bc == '0'; ++bc) {}
if (bc == rhs) {
++bc;
++rhs;
}
}
// Compare the epoch
auto res = cmpFragment (ac, lhs, bc, rhs);
if (res != 0)
return res;
// Skip the :
if (lhs != ac)
lhs++;
if (rhs != bc)
rhs++;
// Find the last -
auto dlhs = cast(immutable(char)*) memrchr (lhs, '-', aEnd - lhs);
auto drhs = cast(immutable(char)*) memrchr (rhs, '-', bEnd - rhs);
if (dlhs is null)
dlhs = aEnd;
if (drhs is null)
drhs = bEnd;
// Compare the main version
res = cmpFragment (lhs, dlhs, rhs, drhs);
if (res != 0)
return res;
// Skip the -
if (dlhs != lhs)
dlhs++;
if (drhs != rhs)
drhs++;
// no debian revision need to be treated like -0
if (*(dlhs - 1) == '-' && *(drhs - 1) == '-') {
return cmpFragment (dlhs, aEnd, drhs, bEnd);
} else if (*(dlhs - 1) == '-') {
immutable(char)* zeroZ = "0";
return cmpFragment (dlhs, aEnd, zeroZ, zeroZ + 1);
} else if (*(drhs - 1) == '-') {
immutable(char)* zeroZ = "0";
return cmpFragment (zeroZ, zeroZ + 1, drhs, bEnd);
} else {
return 0;
}
}
unittest
{
import std.stdio : writeln;
writeln ("TEST: ", "DebianUtils");
assert (compareVersions ("6", "8") < 0);
assert (compareVersions ("0.6.12b-d", "0.6.12a") > 0);
assert (compareVersions ("7.4", "7.4") == 0);
assert (compareVersions ("ab.d", "ab.f") < 0);
assert (compareVersions ("0.6.16", "0.6.14") > 0);
assert (compareVersions ("3.0.rc2", "3.0.0") > 0);
assert (compareVersions ("3.0.0~rc2", "3.0.0") < 0);
assert (compareVersions ("4:5.6-2", "8.0-6") > 0);
assert (compareVersions ("1:1.0-4", "3:0.8-2") < 0);
}
appstream-generator-0.8.6/src/asgen/backends/debian/package.d 0000664 0000000 0000000 00000001647 14160667064 0024147 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.debian;
public import asgen.backends.debian.debpkg;
public import asgen.backends.debian.debpkgindex;
appstream-generator-0.8.6/src/asgen/backends/debian/tagfile.d 0000664 0000000 0000000 00000011022 14160667064 0024153 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2018 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.debian.tagfile;
import std.stdio : File;
import std.string : startsWith, indexOf, chompPrefix, strip, split, splitLines;
import std.typecons : Flag, Yes;
import std.array : appender, empty;
import std.conv : to;
import std.path : buildPath;
import asgen.zarchive;
import asgen.logging;
/**
* Parser for Debian's RFC2822-style metadata.
*/
final class TagFile
{
private:
string[] content;
uint pos;
string[string] currentBlock;
string _fname;
public:
this () @trusted
{
currentBlock.clear ();
}
void open (string fname, Flag!"compressed" compressed = Yes.compressed) @trusted
{
_fname = fname;
if (compressed) {
auto data = decompressFile (fname);
load (data);
} else {
import std.stdio;
auto f = File (fname, "r");
auto data = appender!string;
string line;
while ((line = f.readln ()) !is null)
data ~= line;
load (data.data);
}
}
@property string fname () const { return _fname; }
void load (string data)
{
content = data.splitLines ();
pos = 0;
readCurrentBlockData ();
}
void first () {
pos = 0;
}
private void readCurrentBlockData () @trusted
{
currentBlock.clear ();
immutable clen = content.length;
for (auto i = pos; i < clen; i++) {
if (content[i] == "")
break;
// check whether we are in a multiline value field, and just skip forward in that case
if (startsWith (content[i], " "))
continue;
immutable separatorIndex = indexOf (content[i], ':');
if (separatorIndex <= 0)
continue; // this is no field
auto fieldName = content[i][0..separatorIndex];
auto fdata = content[i][separatorIndex+1..$];
if ((i+1 >= clen)
|| (!startsWith (content[i+1], " "))) {
// we have a single-line field
currentBlock[fieldName] = fdata.strip ();
} else {
// we have a multi-line field
auto fdata_ml = appender!string ();
fdata_ml ~= fdata.strip ();
for (auto j = i+1; j < clen; j++) {
auto slice = chompPrefix (content[j], " ");
if (slice == content[j])
break;
if (fdata_ml.data == "") {
fdata_ml = appender!string ();
fdata_ml ~= slice;
} else {
fdata_ml ~= "\n";
fdata_ml ~= slice;
}
}
currentBlock[fieldName] = fdata_ml.data;
}
}
}
bool nextSection () @trusted
{
bool breakNext = false;
immutable clen = content.length;
currentBlock.clear ();
if (pos >= clen)
return false;
uint i;
for (i = pos; i < clen; i++) {
if (content[i] == "") {
pos = i + 1;
breakNext = true;
} else if (breakNext) {
break;
}
}
// check if we reached the end of this file
if (i == clen)
pos = cast(uint) clen;
if (pos >= clen)
return false;
readCurrentBlockData ();
return true;
}
string readField (string name, string defaultValue = null) @trusted
{
auto dataP = name in currentBlock;
if (dataP is null)
return defaultValue; // we found nothing
else
return *dataP;
}
}
appstream-generator-0.8.6/src/asgen/backends/dummy/ 0000775 0000000 0000000 00000000000 14160667064 0022310 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/src/asgen/backends/dummy/dummypkg.d 0000664 0000000 0000000 00000004757 14160667064 0024327 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.dummy.dummypkg;
import std.stdio;
import std.string;
import std.array : empty;
import asgen.backends.interfaces;
import asgen.logging;
final class DummyPackage : Package
{
private:
string pkgname;
string pkgver;
string pkgarch;
string pkgmaintainer;
string[string] desc;
string testPkgFname;
PackageKind _kind;
public:
@property override string name () const { return pkgname; }
@property override string ver () const { return pkgver; }
@property override string arch () const { return pkgarch; }
@property override const(string[string]) description () const { return desc; }
override
@property string getFilename () const { return testPkgFname; }
@property void filename (string fname) { testPkgFname = fname; }
override
@property string maintainer () const { return pkgmaintainer; }
@property void maintainer (string maint) { pkgmaintainer = maint; }
this (string pname, string pver, string parch)
{
pkgname = pname;
pkgver = pver;
pkgarch = parch;
_kind = PackageKind.PHYSICAL;
}
void setDescription (string text, string locale)
{
desc[locale] = text;
}
override
ubyte[] getFileData (string fname)
{
if (fname == "TEST")
return ['N', 'O', 'T', 'H', 'I', 'N', 'G'];
return [];
}
@property override
string[] contents ()
{
return ["NOTHING1", "NOTHING2"];
}
override
void finish ()
{
}
@property override
PackageKind kind () @safe pure
{
return this._kind;
}
@property
void kind (PackageKind v) @safe pure
{
this._kind = v;
}
}
appstream-generator-0.8.6/src/asgen/backends/dummy/package.d 0000664 0000000 0000000 00000001643 14160667064 0024054 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.dummy;
public import asgen.backends.dummy.dummypkg;
public import asgen.backends.dummy.pkgindex;
appstream-generator-0.8.6/src/asgen/backends/dummy/pkgindex.d 0000664 0000000 0000000 00000003171 14160667064 0024270 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.dummy.pkgindex;
import std.stdio;
import std.path;
import std.string;
import std.algorithm : remove;
import asgen.logging;
import asgen.backends.interfaces;
import asgen.backends.dummy.dummypkg;
final class DummyPackageIndex : PackageIndex
{
private:
Package[][string] pkgCache;
public:
this (string dir)
{
}
void release ()
{
pkgCache = null;
}
Package[] packagesFor (string suite, string section, string arch, bool withLongDescs = true)
{
return [new DummyPackage ("test", "1.0", "amd64")];
}
Package packageForFile (string fname, string suite = null, string section = null)
{
return null; // FIXME: not implemented
}
bool hasChanges (DataStore dstore, string suite, string section, string arch)
{
return true;
}
}
appstream-generator-0.8.6/src/asgen/backends/interfaces.d 0000664 0000000 0000000 00000016016 14160667064 0023451 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2017 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.interfaces;
import std.typecons : Nullable;
import std.string : format;
import std.array : empty;
import appstream.Component;
import glib.KeyFile : KeyFile;
public import asgen.datastore;
class GStreamer
{
immutable string[] decoders;
immutable string[] encoders;
immutable string[] elements;
immutable string[] uri_sinks;
immutable string[] uri_sources;
@property @safe pure bool isNotEmpty () const {
return !(decoders.empty &&
encoders.empty &&
elements.empty &&
uri_sinks.empty &&
uri_sources.empty);
}
this () {
decoders = encoders = elements = uri_sinks = uri_sources = [];
}
this (immutable string[] decoders,
immutable string[] encoders,
immutable string[] elements,
immutable string[] uri_sinks,
immutable string[] uri_sources) {
this.decoders = decoders;
this.encoders = encoders;
this.elements = elements;
this.uri_sinks = uri_sinks;
this.uri_sources = uri_sources;
}
}
/**
* Type of a package that can be processed.
* Allows distinguishing "real" packages from
* virtual or fake ones that are used internally.
*/
enum PackageKind
{
UNKNOWN,
PHYSICAL,
FAKE
}
/**
* Represents a distribution package in the generator.
*/
abstract class Package
{
@property string name () const @safe pure;
@property string ver () const @safe pure;
@property string arch () const @safe pure;
@property string maintainer () const;
/**
* Type of this package (whether it actually exists or is a fake/virtual package)
* You pretty much always want PHYSICAL.
*/
@property PackageKind kind () @safe pure
{
return PackageKind.PHYSICAL;
}
/**
* A associative array containing package descriptions.
* Key is the language (or locale), value the description.
*
* E.g.: ["en": "A description.", "de": "Eine Beschreibung"]
*/
@property const(string[string]) description () const;
/**
* A associative array containing package summaries.
* Key is the language (or locale), value the summary.
*
* E.g.: ["en": "foo the bar"]
*/
@property const(string[string]) summary () const { return (string[string]).init; }
/**
* Local filename of the package. This string is only used for
* issue reporting and other information, the file is never
* accessed directly (all data is retrieved via getFileData()).
*
* This function should return a local filepath, backends might
* download missing packages on-demand from a web location.
*/
string getFilename ();
/**
* A list payload files this package contains.
*/
@property string[] contents ();
/**
* Obtain data for a specific file in the package.
*/
abstract const(ubyte)[] getFileData (string fname);
/**
* Remove temporary data that might have been created while loading information from
* this package. This function can be called to avoid excessive use of disk space.
* As opposed to `close()`, the package may be reopened afterwards.
*/
void cleanupTemp () {}
/**
* Close the package. This function is called when we will
* no longer request any file data from this package.
*/
abstract void finish () {}
@property Nullable!GStreamer gst () { return Nullable!GStreamer (); }
/**
* Retrieve backend-specific translations.
*
* (currently only used by the Ubuntu backend)
*/
string[string] getDesktopFileTranslations (KeyFile desktopFile, const string text) { return null; }
@property bool hasDesktopFileTranslations () const { return false; }
private string pkid;
/**
* Get the unique identifier for this package.
* The ID is supposed to be unique per backend, it should never appear
* multiple times in suites/sections.
*/
@property
final string id () @safe pure
{
import std.array : empty;
if (pkid.empty)
pkid = "%s/%s/%s".format (this.name, this.ver, this.arch);
return pkid;
}
/**
* Check if the package is valid.
* A Package must at least have a name, version and architecture defined.
*/
@safe pure
final bool isValid ()
{
import std.array : empty;
return (!name.empty) &&
(!ver.empty) &&
(!arch.empty);
}
@safe pure override
string toString ()
{
return id;
}
}
/**
* An index of information about packages in a distribution.
*/
interface PackageIndex
{
/**
* Called after a set of operations has completed, which allows the index to
* release memory it might have allocated for cached data, or delete temporary
* files.
**/
void release ();
/**
* Get a list of packages for the given suite/section/arch triplet.
* The PackageIndex should cache the data if obtaining it is an expensive
* operation, since the generator might query the data multiple times.
**/
Package[] packagesFor (string suite,
string section,
string arch,
bool withLongDescs = true);
/**
* Get an abstract package representation for a physical package
* file. A suite name and section name is obviously given.
* This function is used in case only processing of one particular
* package is requested.
* Backends should return null if the feature is not implemented.
**/
Package packageForFile (string fname,
string suite = null,
string section = null);
/**
* Check if the index for the given suite/section/arch triplet has changed since
* the last generator run. The index can use the (get/set)RepoInfo methods on DataCache
* to store mtime or checksum data for the given suite.
* For the lifetime of the PackagesIndex, this method must return the same result,
* which means an internal cache is useful.
*/
bool hasChanges (DataStore dstore,
string suite,
string section,
string arch);
}
appstream-generator-0.8.6/src/asgen/backends/rpmmd/ 0000775 0000000 0000000 00000000000 14160667064 0022274 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/src/asgen/backends/rpmmd/package.d 0000664 0000000 0000000 00000000172 14160667064 0024034 0 ustar 00root root 0000000 0000000
module asgen.backends.rpmmd;
public import asgen.backends.rpmmd.rpmpkg;
public import asgen.backends.rpmmd.rpmpkgindex;
appstream-generator-0.8.6/src/asgen/backends/rpmmd/rpmpkg.d 0000664 0000000 0000000 00000005066 14160667064 0023750 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2017 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.rpmmd.rpmpkg;
import std.stdio;
import std.string;
import std.array : empty;
import asgen.logging;
import asgen.zarchive;
import asgen.backends.interfaces;
final class RPMPackage : Package
{
private:
string pkgname;
string pkgver;
string pkgarch;
string pkgmaintainer;
string[string] desc;
string[string] summ;
string pkgFname;
string[] contentsL;
ArchiveDecompressor archive;
public:
override
@property string name () const { return pkgname; }
@property void name (string val) { pkgname = val; }
override
@property string ver () const { return pkgver; }
@property void ver (string val) { pkgver = val; }
override
@property string arch () const { return pkgarch; }
@property void arch (string val) { pkgarch = val; }
override
@property const(string[string]) description () const { return desc; }
override
@property string getFilename () const { return pkgFname; }
@property void filename (string fname) { pkgFname = fname; }
override
@property string maintainer () const { return pkgmaintainer; }
@property void maintainer (string maint) { pkgmaintainer = maint; }
void setDescription (string text, string locale)
{
desc[locale] = text;
}
void setSummary (string text, string locale)
{
summ[locale] = text;
}
override
const(ubyte)[] getFileData (string fname)
{
if (!archive.isOpen)
archive.open (this.getFilename);
return archive.readData (fname);
}
@property override
string[] contents ()
{
return contentsL;
}
@property
void contents (string[] c)
{
contentsL = c;
}
override
void finish ()
{
}
}
appstream-generator-0.8.6/src/asgen/backends/rpmmd/rpmpkgindex.d 0000664 0000000 0000000 00000016332 14160667064 0024776 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2017 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.rpmmd.rpmpkgindex;
import std.stdio : writeln;
import std.path : buildPath;
import std.array : appender, empty;
import std.string : format;
import std.algorithm : canFind, endsWith;
import std.conv : to;
import std.xml;
static import std.file;
import asgen.logging;
import asgen.backends.interfaces;
import asgen.backends.rpmmd.rpmpkg;
final class RPMPackageIndex : PackageIndex
{
private:
string rootDir;
Package[][string] pkgCache;
public:
this (string dir)
{
this.rootDir = dir;
if (!std.file.exists (dir))
throw new Exception ("Directory '%s' does not exist.", dir);
}
void release ()
{
pkgCache = null;
}
private void setPkgDescription (RPMPackage pkg, string pkgDesc)
{
if (pkgDesc is null)
return;
auto desc = "
%s
".format (pkgDesc);
pkg.setDescription (desc, "C");
}
private RPMPackage[] loadPackages (string suite, string section, string arch)
{
auto repoRoot = buildPath (rootDir, suite, section, arch, "os");
auto primaryIndexFiles = appender!(string[]);
auto filelistFiles = appender!(string[]);
auto repoMdIndexContent = cast(string) std.file.read (buildPath (repoRoot, "repodata", "repomd.xml"));
auto indexXml = new DocumentParser (repoMdIndexContent);
indexXml.onStartTag["data"] = (ElementParser xml) {
immutable dataType = xml.tag.attr["type"];
if (dataType == "primary") {
xml.onStartTag["location"] = (ElementParser x) {
primaryIndexFiles ~= x.tag.attr["href"];
};
xml.parse ();
} else if (dataType == "filelists") {
xml.onStartTag["location"] = (ElementParser x) {
filelistFiles ~= x.tag.attr["href"];
};
xml.parse ();
}
};
indexXml.parse();
// package-id -> RPMPackage
RPMPackage[string] pkgMap;
// parse the primary metadata
foreach (ref primaryFile; primaryIndexFiles.data) {
immutable metaFname = buildPath (repoRoot, primaryFile);
string data;
if (primaryFile.endsWith (".xml")) {
data = cast(string) std.file.read (metaFname);
} else {
import asgen.zarchive : decompressFile;
data = decompressFile (metaFname);
}
auto pkgXml = new DocumentParser (data);
pkgXml.onStartTag["package"] = (ElementParser xml) {
// make sure we only check RPM packages
if (xml.tag.attr["type"] != "rpm")
return;
auto pkg = new RPMPackage;
pkg.maintainer = "None";
xml.onEndTag["name"] = (in Element e) { pkg.name = e.text; };
xml.onEndTag["arch"] = (in Element e) { pkg.arch = e.text; };
xml.onEndTag["summary"] = (in Element e) { pkg.setSummary (e.text, "C"); };
xml.onEndTag["description"] = (in Element e) { pkg.setDescription (e.text, "C"); };
xml.onEndTag["packager"] = (in Element e) { pkg.maintainer = e.text; };
xml.onStartTag["version"] = (ElementParser x) {
immutable epoch = x.tag.attr["epoch"];
immutable upstream_ver = x.tag.attr["ver"];
immutable rel = x.tag.attr["rel"];
if ((epoch == "0") || (epoch.empty))
pkg.ver = "%s-%s".format (upstream_ver, rel);
else
pkg.ver = "%s:%s-%s".format (epoch, upstream_ver, rel);
};
xml.onStartTag["location"] = (ElementParser x) { pkg.filename = buildPath (repoRoot, x.tag.attr["href"]); };
string pkgidCS;
xml.onEndTag["checksum"] = (in Element e) {
// we are only interested in the package-id here
if (e.tag.attr["pkgid"] != "YES")
return;
pkgidCS = e.text;
};
xml.parse ();
if (pkgidCS.empty) {
logWarning ("Found package '%s' in '%s' without suitable pkgid. Ignoring it.", pkg.name, primaryFile);
return;
}
pkgMap[pkgidCS] = pkg;
};
pkgXml.parse();
}
pkgMap.rehash;
// read the filelists
foreach (ref filelistFile; filelistFiles.data) {
immutable flistFname = buildPath (repoRoot, filelistFile);
string data;
if (filelistFile.endsWith (".xml")) {
data = cast(string) std.file.read (flistFname);
} else {
import asgen.zarchive : decompressFile;
data = decompressFile (flistFname);
}
auto flXml = new DocumentParser (data);
flXml.onStartTag["package"] = (ElementParser xml) {
immutable pkgid = xml.tag.attr["pkgid"];
auto pkgP = pkgid in pkgMap;
if (pkgP is null)
return;
auto pkg = *pkgP;
auto contents = appender!(string[]);
xml.onEndTag["file"] = (in Element e) { contents ~= e.text; };
xml.parse ();
pkg.contents = contents.data;
};
flXml.parse();
}
return pkgMap.values;
}
Package[] packagesFor (string suite, string section, string arch, bool withLongDescs = true)
{
immutable id = "%s-%s-%s".format (suite, section, arch);
if (id !in pkgCache) {
auto pkgs = loadPackages (suite, section, arch);
synchronized (this) pkgCache[id] = to!(Package[]) (pkgs);
}
return pkgCache[id];
}
Package packageForFile (string fname, string suite = null, string section = null)
{
return null; // FIXME: not implemented
}
bool hasChanges (DataStore dstore, string suite, string section, string arch)
{
return true;
}
}
unittest {
import std.algorithm.sorting : sort;
import asgen.utils : getTestSamplesDir;
writeln ("TEST: ", "RpmMDPackageIndex");
auto pi = new RPMPackageIndex (buildPath (getTestSamplesDir (), "rpmmd"));
auto pkgs = pi.loadPackages ("26", "Workstation", "x86_64");
assert (pkgs.length == 4);
}
appstream-generator-0.8.6/src/asgen/backends/ubuntu/ 0000775 0000000 0000000 00000000000 14160667064 0022477 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/src/asgen/backends/ubuntu/package.d 0000664 0000000 0000000 00000001647 14160667064 0024247 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.ubuntu;
public import asgen.backends.ubuntu.ubupkg;
public import asgen.backends.ubuntu.ubupkgindex;
appstream-generator-0.8.6/src/asgen/backends/ubuntu/ubupkg.d 0000664 0000000 0000000 00000016353 14160667064 0024151 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016 Canonical Ltd
* Author: Iain Lane
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.ubuntu.ubupkg;
import std.path : buildPath;
import std.file : mkdirRecurse;
import std.conv : to;
import glib.Internationalization;
import glib.KeyFile;
import appstream.Component;
import asgen.logging;
import asgen.utils : DESKTOP_GROUP;
import asgen.backends.debian.debpkg : DebPackage;
import asgen.backends.interfaces;
extern (C) char *bindtextdomain (const(char*) domainname, const(char*) dirName) nothrow @nogc;
/**
* A helper class that provides functions to work with language packs
* used in Ubuntu.
*/
final class LanguagePackProvider
{
private:
UbuntuPackage[] langpacks;
string globalTmpDir;
string langpackDir;
string localeDir;
string[] langpackLocales;
public:
this (string globalTmpDir)
{
this.globalTmpDir = globalTmpDir;
this.langpackDir = buildPath (globalTmpDir, "langpacks");
this.localeDir = buildPath (langpackDir, "locales");
}
void addLanguagePacks (UbuntuPackage[] langpacks)
{
synchronized (this)
this.langpacks ~= langpacks;
}
void clear ()
{
synchronized (this)
this.langpacks = [];
}
private void extractLangpacks ()
{
import std.algorithm : filter, map;
import std.array : appender, array, split;
import std.file : dirEntries, exists, SpanMode, readText;
import std.parallelism : parallel;
import std.path : baseName;
import std.process : Pid, spawnProcess, wait;
import std.string : splitLines, startsWith;
import glib.Util : Util;
if (!langpackDir.exists) {
bool[string] extracted;
langpackDir.mkdirRecurse ();
foreach (ref pkg; langpacks) {
if (pkg.name in extracted)
continue;
logDebug ("Extracting %s", pkg.name);
pkg.extractPackage (langpackDir);
extracted[pkg.name] = true;
}
auto supportedd = buildPath (langpackDir, "var", "lib", "locales", "supported.d");
localeDir.mkdirRecurse ();
foreach (locale; parallel (supportedd.dirEntries (SpanMode.shallow), 5)) {
foreach (ref line; locale.readText.splitLines) {
auto components = line.split (" ");
auto localecharset = components[0].split (".");
auto outdir = buildPath (localeDir, components[0]);
logDebug ("Generating locale in %s", outdir);
auto pid = spawnProcess ([Util.findProgramInPath ("localedef"),
"--no-archive",
"-i",
localecharset[0],
"-c",
"-f",
components[1],
outdir]);
scope (exit) wait (pid);
}
}
}
// we don't need it; we've already extracted the langpacks
langpacks = [];
if (langpackLocales is null)
langpackLocales = localeDir.dirEntries (SpanMode.shallow)
.filter!(f => f.isDir)
.map!(f => f.name.baseName)
.array;
}
private auto getTranslationsPrivate (const string domain, const string text)
{
import core.stdc.locale : setlocale, LC_ALL;
import core.stdc.string : strdup;
import core.sys.posix.stdlib : getenv, setenv, unsetenv;
import std.string : toStringz;
char *[char *] env;
foreach (ref var; ["LC_ALL", "LANG", "LANGUAGE", "LC_MESSAGES"]) {
const value = getenv (var.toStringz);
if (value !is null) {
env[var.toStringz] = getenv (var.toStringz).strdup;
unsetenv (var.toStringz);
}
}
scope (exit) {
foreach (key, val; env)
setenv (key, val, false);
}
setenv ("LOCPATH", localeDir.toStringz, true);
auto initialLocale = setlocale (LC_ALL, "");
scope(exit) setlocale (LC_ALL, initialLocale);
auto dir = buildPath (langpackDir,
"usr",
"share",
"locale-langpack");
string[string] ret;
foreach (ref locale; langpackLocales) {
setlocale (LC_ALL, locale.toStringz);
bindtextdomain (domain.toStringz, dir.toStringz);
const translatedtext = Internationalization.dgettext (domain, text);
if (text != translatedtext)
ret[locale] = translatedtext;
}
return ret;
}
string[string] getTranslations (const string domain, const string text)
{
// this functions do nasty things like changing environment variables and
// messing with other global state. We therefore need to ensure that nothing
// else is running in parallel.
synchronized {
extractLangpacks ();
return getTranslationsPrivate (domain, text);
}
}
}
/**
* An Ubuntu package.
*/
final class UbuntuPackage : DebPackage
{
private:
LanguagePackProvider lpack;
public:
this (string pname, string pver, string parch, LanguagePackProvider lpack)
{
this.lpack = lpack;
super (pname, pver, parch);
assert (this.lpack !is null);
updateTmpDirPath ();
}
@property override
bool hasDesktopFileTranslations () const { return true; }
override
string[string] getDesktopFileTranslations (KeyFile desktopFile, const string text)
{
string langpackdomain;
try {
langpackdomain = desktopFile.getString (DESKTOP_GROUP,
"X-Ubuntu-Gettext-Domain");
} catch (Exception) {
try {
langpackdomain = desktopFile.getString (DESKTOP_GROUP,
"X-GNOME-Gettext-Domain");
} catch (Exception) {
return null;
}
}
logDebug ("%s has langpack domain %s", name, langpackdomain);
return lpack.getTranslations (langpackdomain, text);
}
}
appstream-generator-0.8.6/src/asgen/backends/ubuntu/ubupkgindex.d 0000664 0000000 0000000 00000006221 14160667064 0025172 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016 Canonical Ltd
* Author: Iain Lane
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.backends.ubuntu.ubupkgindex;
import std.string : format;
import std.array : appender;
import std.conv : to;
import asgen.backends.debian;
import asgen.backends.interfaces;
import asgen.backends.ubuntu.ubupkg;
final class UbuntuPackageIndex : DebianPackageIndex
{
private:
LanguagePackProvider langpacks;
// holds the IDs of suite/section/arch combinations where we scanned language packs
bool[string] checkedLangPacks;
public:
this (string dir)
{
super (dir);
/*
* UbuntuPackage needs to extract the langpacks, so we give it an array
* of langpacks. There is a small overhead when computing this array
* which might be unnecessary if no processed packages are using
* langpacks, but otherwise we need to keep a reference to all packages
* around, which is very expensive.
*/
langpacks = new LanguagePackProvider (tmpDir);
}
override
void release ()
{
super.release ();
checkedLangPacks.clear ();
langpacks = new LanguagePackProvider (tmpDir);
}
override protected
DebPackage newPackage (string name, string ver, string arch)
{
return new UbuntuPackage (name, ver, arch, langpacks);
}
override
Package[] packagesFor (string suite, string section, string arch, bool withLongDescs = true)
{
import std.string : startsWith;
auto pkgs = super.packagesFor (suite, section, arch, withLongDescs);
immutable ssaId = "%s/%s/%s".format (suite, section, arch);
if (ssaId in checkedLangPacks)
return pkgs; // no need to scan for language packs, we already did that
// scan for language packs and add them to the data provider
synchronized (this) {
auto pkgslangpacks = appender!(UbuntuPackage[]);
pkgslangpacks.reserve (32);
foreach (ref pkg; pkgs) {
if (pkg.name.startsWith ("language-pack-"))
pkgslangpacks ~= pkg.to!UbuntuPackage;
}
langpacks.addLanguagePacks (pkgslangpacks.data);
checkedLangPacks[ssaId] = true;
}
return pkgs;
}
override
Package packageForFile (string fname, string suite = null, string section = null)
{
return null; // FIXME: not implemented
}
}
appstream-generator-0.8.6/src/asgen/bindings/ 0000775 0000000 0000000 00000000000 14160667064 0021200 5 ustar 00root root 0000000 0000000 appstream-generator-0.8.6/src/asgen/bindings/libarchive.d 0000664 0000000 0000000 00000007657 14160667064 0023474 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This library is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this library. If not, see .
*/
module asgen.bindings.libarchive;
import core.stdc.stdio;
import std.conv : octal;
extern(C):
nothrow:
@nogc:
static if (!is(typeof(usize))) private alias usize = size_t;
struct archive {}
struct archive_entry {}
immutable ARCHIVE_EOF = 1; /* Found end of archive. */
immutable ARCHIVE_OK = 0; /* Operation was successful. */
immutable ARCHIVE_RETRY = -10; /* Retry might succeed. */
immutable ARCHIVE_WARN = -20; /* Partial success. */
immutable ARCHIVE_FAILED = -25; /* Current operation cannot complete. */
immutable ARCHIVE_FATAL = -30; /* No more operations are possible. */
immutable AE_IFMT = octal!170000;
immutable AE_IFREG = octal!100000;
immutable AE_IFLNK = octal!120000;
immutable AE_IFSOCK = octal!140000;
immutable AE_IFCHR = octal!20000;
immutable AE_IFBLK = octal!60000;
immutable AE_IFDIR = octal!40000;
immutable AE_IFIFO = octal!10000;
const(char) *archive_error_string (archive*);
int archive_errno (archive*);
archive *archive_read_new ();
int archive_read_free (archive*);
int archive_read_support_filter_all (archive*);
int archive_read_support_filter_gzip (archive*);
int archive_read_support_filter_lzma (archive*);
int archive_read_support_format_raw (archive*);
int archive_read_support_format_empty (archive*);
int archive_read_support_format_all (archive*);
int archive_read_support_format_ar (archive*);
int archive_read_support_format_gnutar (archive*);
int archive_write_set_filter_option (archive *a, const(char) *m, const(char) *o, const(char) *v);
int archive_read_open_filename (archive*, const(char) *filename, usize block_size);
int archive_read_open_FILE (archive*, FILE *file);
int archive_read_open_memory (archive*, void *buff, size_t size);
ptrdiff_t archive_read_data (archive*, const(void*), usize);
int archive_read_next_header (archive*, archive_entry**);
int archive_read_data_skip (archive*);
int archive_read_data_block (archive *a, const(void*) *buff, size_t *size, long *offset);
archive_entry *archive_entry_new ();
void archive_entry_free (archive_entry*);
const(char) *archive_entry_pathname (archive_entry*);
void archive_entry_set_pathname (archive_entry*, const(char) *);
uint archive_entry_filetype (archive_entry*);
void archive_entry_set_size (archive_entry*, long);
void archive_entry_set_filetype (archive_entry*, uint);
void archive_entry_set_perm (archive_entry*, uint);
void archive_entry_set_mtime (archive_entry*, ulong sec, long nsec);
const(char) *archive_entry_symlink (archive_entry*);
archive *archive_write_new ();
int archive_write_free (archive*);
int archive_write_close (archive*);
int archive_write_add_filter_gzip (archive*);
int archive_write_add_filter_xz (archive*);
int archive_write_set_format_pax (archive*);
int archive_write_set_format_pax_restricted (archive*);
int archive_write_set_format_raw (archive*);
//int archive_write_set_format_raw (archive *a); /// Will be available in the next version of libarchive (to be released in 2016)
int archive_write_set_format_by_name (archive*, const(char) *name);
int archive_write_open_filename (archive*, const(char) *file);
int archive_write_header (archive*, archive_entry*);
size_t archive_write_data(archive*, const(void)*, size_t);
appstream-generator-0.8.6/src/asgen/bindings/lmdb.d 0000664 0000000 0000000 00000015652 14160667064 0022274 0 ustar 00root root 0000000 0000000 /*
* @author Howard Chu, Symas Corporation.
*
* @copyright Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* .
*
* @par Derived From:
* This code is derived from btree.c written by Martin Hedenfalk.
*
* Copyright (c) 2009, 2010 Martin Hedenfalk
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
module asgen.bindings.lmdb;
// ////////////////////////////////////////////////////////////////////////// //
// C API
extern(C):
nothrow:
@nogc:
static if (!is(typeof(usize))) private alias usize = size_t;
alias mdb_mode_t = uint;
struct mdb_filehandle_ts {}
alias mdb_filehandle_t = mdb_filehandle_ts*;
enum {
MDB_VERSION_MAJOR = 0,
MDB_VERSION_MINOR = 9,
MDB_VERSION_PATCH = 27,
MDB_VERSION_DATE = "October 26, 2020",
}
struct MDB_env_s {}
alias MDB_envp = MDB_env_s*;
struct MDB_txn_s {}
alias MDB_txnp = MDB_txn_s*;
alias MDB_dbi = uint;
struct MDB_cursor_s {}
alias MDB_cursorp = MDB_cursor_s*;
struct MDB_val {
usize mv_size;
void* mv_data;
}
enum {
MDB_FIXEDMAP = 0x01,
MDB_NOSUBDIR = 0x4000,
MDB_NOSYNC = 0x10000,
MDB_RDONLY = 0x20000,
MDB_NOMETASYNC = 0x40000,
MDB_WRITEMAP = 0x80000,
MDB_MAPASYNC = 0x100000,
MDB_NOTLS = 0x200000,
MDB_NOLOCK = 0x400000,
MDB_NORDAHEAD = 0x800000,
MDB_NOMEMINIT = 0x1000000
}
enum {
MDB_REVERSEKEY = 0x02,
MDB_DUPSORT = 0x04,
MDB_INTEGERKEY = 0x08,
MDB_DUPFIXED = 0x10,
MDB_INTEGERDUP = 0x20,
MDB_REVERSEDUP = 0x40,
MDB_CREATE = 0x40000
}
enum {
MDB_NOOVERWRITE = 0x10,
MDB_NODUPDATA = 0x20,
MDB_RESERVE = 0x10000,
MDB_APPEND = 0x20000,
MDB_APPENDDUP = 0x40000,
MDB_MULTIPLE = 0x80000
}
enum /*MDB_cursor_op*/ {
MDB_FIRST,
MDB_FIRST_DUP,
MDB_GET_BOTH,
MDB_GET_BOTH_RANGE,
MDB_GET_CURRENT,
MDB_GET_MULTIPLE,
MDB_LAST,
MDB_LAST_DUP,
MDB_NEXT,
MDB_NEXT_DUP,
MDB_NEXT_MULTIPLE,
MDB_NEXT_NODUP,
MDB_PREV,
MDB_PREV_DUP,
MDB_PREV_NODUP,
MDB_SET,
MDB_SET_KEY,
MDB_SET_RANGE,
}
enum {
MDB_SUCCESS = 0,
MDB_KEYEXIST = (-30799),
MDB_NOTFOUND = (-30798),
MDB_PAGE_NOTFOUND = (-30797),
MDB_CORRUPTED = (-30796),
MDB_PANIC = (-30795),
MDB_VERSION_MISMATCH = (-30794),
MDB_INVALID = (-30793),
MDB_MAP_FULL = (-30792),
MDB_DBS_FULL = (-30791),
MDB_READERS_FULL = (-30790),
MDB_TLS_FULL = (-30789),
MDB_TXN_FULL = (-30788),
MDB_CURSOR_FULL = (-30787),
MDB_PAGE_FULL = (-30786),
MDB_MAP_RESIZED = (-30785),
MDB_INCOMPATIBLE = (-30784),
MDB_BAD_RSLOT = (-30783),
MDB_BAD_TXN = (-30782),
MDB_BAD_VALSIZE = (-30781),
MDB_BAD_DBI = (-30780),
MDB_LAST_ERRCODE = MDB_BAD_DBI
}
struct MDB_stat {
uint ms_psize;
uint ms_depth;
usize ms_branch_pages;
usize ms_leaf_pages;
usize ms_overflow_pages;
usize ms_entries;
}
struct MDB_envinfo {
void* me_mapaddr;
usize me_mapsize;
usize me_last_pgno;
usize me_last_txnid;
uint me_maxreaders;
uint me_numreaders;
}
const(char)* mdb_version (int* major, int* minor, int* patch);
const(char)* mdb_strerror (int err);
int mdb_env_create (MDB_envp* env);
int mdb_env_open (MDB_envp env, const(char)* path, uint flags, mdb_mode_t mode);
int mdb_env_copy (MDB_envp env, const(char)* path);
int mdb_env_copyfd (MDB_envp env, mdb_filehandle_t fd);
int mdb_env_stat (MDB_envp env, MDB_stat* stat);
int mdb_env_info (MDB_envp env, MDB_envinfo* stat);
int mdb_env_sync (MDB_envp env, int force);
void mdb_env_close (MDB_envp env);
int mdb_env_set_flags (MDB_envp env, uint flags, int onoff);
int mdb_env_get_flags (MDB_envp env, uint* flags);
int mdb_env_get_path (MDB_envp env, const(char)** path);
int mdb_env_get_fd (MDB_envp env, mdb_filehandle_t* fd);
int mdb_env_set_mapsize (MDB_envp env, usize size);
int mdb_env_set_maxreaders (MDB_envp env, uint readers);
int mdb_env_get_maxreaders (MDB_envp env, uint* readers);
int mdb_env_set_maxdbs (MDB_envp env, MDB_dbi dbs);
int mdb_env_get_maxkeysize (MDB_envp env);
int mdb_env_set_userctx (MDB_envp env, void* ctx);
void* mdb_env_get_userctx (MDB_envp env);
int mdb_env_set_assert (MDB_envp env, void function (MDB_envp env, const(char)* msg) func);
int mdb_txn_begin (MDB_envp env, MDB_txnp parent, uint flags, MDB_txnp* txn);
MDB_envp mdb_txn_env (MDB_txnp txn);
usize mdb_txn_id (MDB_txnp txn);
int mdb_txn_commit (MDB_txnp txn);
void mdb_txn_abort (MDB_txnp txn);
void mdb_txn_reset (MDB_txnp txn);
int mdb_txn_renew (MDB_txnp txn);
int mdb_dbi_open (MDB_txnp txn, const(char)* name, uint flags, MDB_dbi* dbi);
int mdb_stat (MDB_txnp txn, MDB_dbi dbi, MDB_stat* stat);
int mdb_dbi_flags (MDB_txnp txn, MDB_dbi dbi, uint* flags);
void mdb_dbi_close (MDB_envp env, MDB_dbi dbi);
int mdb_drop (MDB_txnp txn, MDB_dbi dbi, int del);
int mdb_set_compare (MDB_txnp txn, MDB_dbi dbi, int function (MDB_val* a, MDB_val* b) cmp);
int mdb_set_dupsort (MDB_txnp txn, MDB_dbi dbi, int function (MDB_val* a, MDB_val* b) cmp);
int mdb_set_relfunc (MDB_txnp txn, MDB_dbi dbi, void function (MDB_val* item, void* oldptr, void* newptr, void* relctx) rel);
int mdb_set_relctx (MDB_txnp txn, MDB_dbi dbi, void* ctx);
int mdb_get (MDB_txnp txn, MDB_dbi dbi, MDB_val* key, MDB_val* data);
int mdb_put (MDB_txnp txn, MDB_dbi dbi, MDB_val* key, MDB_val* data, uint flags);
int mdb_del (MDB_txnp txn, MDB_dbi dbi, MDB_val* key, MDB_val* data);
int mdb_cursor_open (MDB_txnp txn, MDB_dbi dbi, MDB_cursorp* cursor);
void mdb_cursor_close (MDB_cursorp cursor);
int mdb_cursor_renew (MDB_txnp txn, MDB_cursorp cursor);
MDB_txnp mdb_cursor_txn (MDB_cursorp cursor);
MDB_dbi mdb_cursor_dbi (MDB_cursorp cursor);
int mdb_cursor_get (MDB_cursorp cursor, MDB_val* key, MDB_val* data, /*MDB_cursor_op*/uint op);
int mdb_cursor_put (MDB_cursorp cursor, MDB_val* key, MDB_val* data, uint flags);
int mdb_cursor_del (MDB_cursorp cursor, uint flags);
int mdb_cursor_count (MDB_cursorp cursor, usize* countp);
int mdb_cmp (MDB_txnp txn, MDB_dbi dbi, MDB_val* a, MDB_val* b);
int mdb_dcmp (MDB_txnp txn, MDB_dbi dbi, MDB_val* a, MDB_val* b);
int mdb_reader_list (MDB_envp env, int function (const(char)* msg, void* ctx) func, void* ctx);
int mdb_reader_check (MDB_envp env, int* dead);
appstream-generator-0.8.6/src/asgen/config.d 0000664 0000000 0000000 00000057012 14160667064 0021022 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2021 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.config;
import std.stdio;
import std.array;
import std.string : format, toLower;
import std.path : dirName, buildPath, buildNormalizedPath, isAbsolute, absolutePath;
import std.conv : to;
import std.json;
import std.typecons;
import std.file : getcwd, thisExePath, exists;
import ascompose.Globals : Globals;
public import appstream.c.types : FormatVersion;
import asgen.utils : existsAndIsDir, randomString, ImageSize;
import asgen.logging;
import asgen.defines : DATADIR;
/**
* Describes a suite in a software repository.
**/
struct Suite
{
string name;
int dataPriority = 0;
string baseSuite;
string iconTheme;
string[] sections;
string[] architectures;
string extraMetainfoDir;
bool isImmutable;
}
/**
* The AppStream metadata type we want to generate.
**/
enum DataType
{
XML,
YAML
}
/**
* Distribution-specific backends.
**/
enum Backend
{
Unknown,
Dummy,
Debian,
Ubuntu,
Archlinux,
RpmMd,
Alpinelinux
}
/**
* Generator features that can be toggled by the user.
*/
struct GeneratorFeatures
{
bool processDesktop;
bool validate;
bool noDownloads;
bool storeScreenshots;
bool optipng;
bool metadataTimestamps;
bool immutableSuites;
bool processFonts;
bool allowIconUpscale;
bool processGStreamer;
bool processLocale;
bool screenshotVideos;
bool propagateMetaInfoArtifacts;
}
/// Fake package name AppStream Generator uses internally to inject additional metainfo on users' request
public immutable EXTRA_METAINFO_FAKE_PKGNAME = "+extra-metainfo";
/// A list of valid icon sizes that we recognize in AppStream
public immutable allowedIconSizes = [ImageSize (48), ImageSize (48, 48, 2),
ImageSize (64), ImageSize (64, 64, 2),
ImageSize (128), ImageSize (128, 128, 2)];
/**
* Policy on a single icon size.
*/
struct IconPolicy
{
ImageSize iconSize; /// Size of the icon this policy is about
bool storeCached; /// True if the icon should be stored in an icon tarball and be cached locally.
bool storeRemote; /// True if this icon should be stored remotely and fetched on demand
bool storeIcon () @property const
{
return storeCached || storeRemote;
}
this (ImageSize size, bool cached, bool remote)
{
iconSize = size;
storeCached = cached;
storeRemote = remote;
}
}
/**
* The global configuration for the metadata generator.
*/
final class Config
{
private:
string workspaceDir;
string exportDir;
string tmpDir;
// thread local
static bool instantiated_;
// thread global
__gshared Config instance_;
this ()
{
import glib.Util : Util;
// our default export format version
formatVersion = FormatVersion.V0_14;
// find all the external binaries we (may) need
// we search for them unconditionally, because the unittests may rely on their absolute
// paths being set even if a particular feature flag that requires them isn't.
optipngBinary = Globals.optipngBinary;
ffprobeBinary = Util.findProgramInPath ("ffprobe");
}
public:
FormatVersion formatVersion;
string projectName;
string archiveRoot;
string mediaBaseUrl;
string htmlBaseUrl;
string backendName;
Backend backend;
Suite[] suites;
string[] oldsuites;
DataType metadataType;
GeneratorFeatures feature; /// Set which features are enabled or disabled
string optipngBinary;
string ffprobeBinary;
bool[string] allowedCustomKeys; // set of allowed keys in tags
string dataExportDir;
string hintsExportDir;
string mediaExportDir;
string htmlExportDir;
long maxScrFileSize;
IconPolicy[] iconSettings;
string caInfo;
static Config get ()
{
if (!instantiated_) {
synchronized (Config.classinfo) {
if (!instance_)
instance_ = new Config ();
instantiated_ = true;
}
}
return instance_;
}
@property
string formatVersionStr ()
{
static import appstream.Utils;
alias AsUtils = appstream.Utils.Utils;
return AsUtils.formatVersionToString (formatVersion);
}
@property
string databaseDir () const
{
return buildPath (workspaceDir, "db");
}
@property
string cacheRootDir () const
{
return buildPath (workspaceDir, "cache");
}
@property
string templateDir () {
// find a suitable template directory
// first check the workspace
auto tdir = buildPath (workspaceDir, "templates");
tdir = getVendorTemplateDir (tdir, true);
if (tdir.empty) {
immutable exeDir = dirName (thisExePath ());
tdir = buildNormalizedPath (exeDir, "..", "..", "..", "data", "templates");
tdir = getVendorTemplateDir (tdir);
if (tdir.empty) {
tdir = getVendorTemplateDir (buildPath (DATADIR, "templates"));
if (tdir.empty) {
tdir = buildNormalizedPath (exeDir, "..", "data", "templates");
tdir = getVendorTemplateDir (tdir);
}
}
}
return tdir;
}
/**
* Helper function to determine a vendor template directory.
*/
private string getVendorTemplateDir (const string dir, bool allowRoot = false) @safe
{
string tdir;
if (!projectName.empty) {
tdir = buildPath (dir, projectName.toLower);
if (existsAndIsDir (tdir))
return tdir;
}
tdir = buildPath (dir, "default");
if (existsAndIsDir (tdir))
return tdir;
if (allowRoot) {
if (existsAndIsDir (dir))
return dir;
}
return null;
}
void loadFromFile (string fname, string enforcedWorkspaceDir = null, string enforcedExportDir = null)
{
// read the configuration JSON file
auto f = File (fname, "r");
string jsonData;
string line;
while ((line = f.readln ()) !is null)
jsonData ~= line;
JSONValue root = parseJSON (jsonData);
if ("WorkspaceDir" in root) {
workspaceDir = root["WorkspaceDir"].str;
} else {
workspaceDir = dirName (fname);
if (workspaceDir.empty)
workspaceDir = getcwd ();
}
// allow overriding the workspace location
if (!enforcedWorkspaceDir.empty)
workspaceDir = enforcedWorkspaceDir;
if (!workspaceDir.isAbsolute)
workspaceDir = workspaceDir.absolutePath;
this.projectName = "Unknown";
if ("ProjectName" in root)
this.projectName = root["ProjectName"].str;
this.archiveRoot = root["ArchiveRoot"].str;
this.mediaBaseUrl = "";
if ("MediaBaseUrl" in root)
this.mediaBaseUrl = root["MediaBaseUrl"].str;
this.htmlBaseUrl = "";
if ("HtmlBaseUrl" in root)
this.htmlBaseUrl = root["HtmlBaseUrl"].str;
// set root export directory
if (enforcedExportDir.empty) {
exportDir = buildPath (workspaceDir, "export");
} else {
exportDir = enforcedExportDir;
logInfo ("Using data export directory root from the command-line: %s", exportDir);
}
if (!exportDir.isAbsolute)
exportDir = exportDir.absolutePath;
// set the default export directory locations, allow people to override them in the config
// (we convert the relative to absolute paths later)
mediaExportDir = "media";
dataExportDir = "data";
hintsExportDir = "hints";
htmlExportDir = "html";
if ("ExportDirs" in root) {
auto edirs = root["ExportDirs"].object;
foreach (dirId; edirs.byKeyValue) {
switch (dirId.key) {
case "Media":
mediaExportDir = dirId.value.str;
break;
case "Data":
dataExportDir = dirId.value.str;
break;
case "Hints":
hintsExportDir = dirId.value.str;
break;
case "Html":
htmlExportDir = dirId.value.str;
break;
default:
logWarning ("Unknown export directory specifier in config: %s", dirId.key);
}
}
}
// convert export directory paths to absolute paths if necessary
mediaExportDir = mediaExportDir.isAbsolute? mediaExportDir : buildNormalizedPath (exportDir, mediaExportDir);
dataExportDir = dataExportDir.isAbsolute? dataExportDir : buildNormalizedPath (exportDir, dataExportDir);
hintsExportDir = hintsExportDir.isAbsolute? hintsExportDir : buildNormalizedPath (exportDir, hintsExportDir);
htmlExportDir = htmlExportDir.isAbsolute? htmlExportDir : buildNormalizedPath (exportDir, htmlExportDir);
// a place where external metainfo data can be injected
auto extraMetainfoDir = buildPath (workspaceDir, "extra-metainfo");
if ("ExtraMetainfoDir" in root)
extraMetainfoDir = root["ExtraMetainfoDir"].str;
this.metadataType = DataType.XML;
if ("MetadataType" in root)
if (root["MetadataType"].str.toLower == "yaml")
this.metadataType = DataType.YAML;
if ("CAInfo" in root)
this.caInfo = root["CAInfo"].str;
// allow specifying the AppStream format version we build data for.
if ("FormatVersion" in root) {
immutable versionStr = root["FormatVersion"].str;
switch (versionStr) {
case "0.8":
formatVersion = FormatVersion.V0_8;
break;
case "0.9":
formatVersion = FormatVersion.V0_9;
break;
case "0.10":
formatVersion = FormatVersion.V0_10;
break;
case "0.11":
formatVersion = FormatVersion.V0_11;
break;
case "0.12":
formatVersion = FormatVersion.V0_12;
break;
default:
logWarning ("Configuration tried to set unknown AppStream format version '%s'. Falling back to default version.", versionStr);
break;
}
}
// we default to the Debian backend for now
auto backendId = "debian";
if ("Backend" in root)
backendId = root["Backend"].str.toLower;
switch (backendId) {
case "dummy":
this.backendName = "Dummy";
this.backend = Backend.Dummy;
this.metadataType = DataType.YAML;
break;
case "debian":
this.backendName = "Debian";
this.backend = Backend.Debian;
this.metadataType = DataType.YAML;
break;
case "ubuntu":
this.backendName = "Ubuntu";
this.backend = Backend.Ubuntu;
this.metadataType = DataType.YAML;
break;
case "arch":
case "archlinux":
this.backendName = "Arch Linux";
this.backend = Backend.Archlinux;
this.metadataType = DataType.XML;
break;
case "mageia":
case "rpmmd":
this.backendName = "RpmMd";
this.backend = Backend.RpmMd;
this.metadataType = DataType.XML;
break;
case "alpinelinux":
this.backendName = "Alpine Linux";
this.backend = Backend.Alpinelinux;
this.metadataType = DataType.XML;
break;
default:
break;
}
auto hasImmutableSuites = false;
foreach (suiteName; root["Suites"].object.byKey) {
Suite suite;
suite.name = suiteName;
// having a suite named "pool" will result in the media pool being copied on
// itself if immutableSuites is used. Since 'pool' is a bad suite name anyway,
// we error out early on this.
if (suiteName == "pool")
throw new Exception ("The name 'pool' is forbidden for a suite.");
auto sn = root["Suites"][suiteName];
if ("dataPriority" in sn)
suite.dataPriority = to!int (sn["dataPriority"].integer);
if ("baseSuite" in sn)
suite.baseSuite = sn["baseSuite"].str;
if ("useIconTheme" in sn)
suite.iconTheme = sn["useIconTheme"].str;
if ("sections" in sn)
foreach (sec; sn["sections"].array)
suite.sections ~= sec.str;
if ("architectures" in sn)
foreach (arch; sn["architectures"].array)
suite.architectures ~= arch.str;
if ("immutable" in sn) {
suite.isImmutable = sn["immutable"].type == JSONType.true_;
if (suite.isImmutable)
hasImmutableSuites = true;
}
const suiteExtraMIDir = buildNormalizedPath (extraMetainfoDir, suite.name);
if (suiteExtraMIDir.existsAndIsDir)
suite.extraMetainfoDir = suiteExtraMIDir;
suites ~= suite;
}
if ("Oldsuites" in root.object) {
import std.algorithm.iteration : map;
oldsuites = map!"a.str"(root["Oldsuites"].array).array;
}
// icon policy
if ("Icons" in root.object) {
import std.algorithm : canFind;
iconSettings.reserve (4);
auto iconsObj = root["Icons"].object;
foreach (iconString; iconsObj.byKey) {
auto iconObj = iconsObj[iconString];
IconPolicy ipolicy;
ipolicy.iconSize = ImageSize (iconString);
if (!allowedIconSizes.canFind (ipolicy.iconSize)) {
logError ("Invalid icon size '%s' selected in configuration, icon policy has been ignored.", iconString);
continue;
}
if (ipolicy.iconSize.width < 0) {
logError ("Malformed icon size '%s' found in configuration, icon policy has been ignored.", iconString);
continue;
}
if ("remote" in iconObj)
ipolicy.storeRemote = iconObj["remote"].type == JSONType.true_;
if ("cached" in iconObj)
ipolicy.storeCached = iconObj["cached"].type == JSONType.true_;
if (ipolicy.storeIcon)
iconSettings ~= ipolicy;
}
// Sanity check
bool defaultSizeFound = false;
foreach (ref ipolicy; iconSettings) {
if (ipolicy.iconSize == ImageSize (64)) {
defaultSizeFound = true;
if (!ipolicy.storeCached) {
logError ("The icon size 64x64 must always be present and be allowed to be cached. Configuration has been adjusted.");
ipolicy.storeCached = true;
break;
}
}
}
if (!defaultSizeFound) {
logError ("The icon size 64x64 must always be present and be allowed to be cached. Configuration has been adjusted.");
IconPolicy ipolicy;
ipolicy.iconSize = ImageSize (64);
ipolicy.storeCached = true;
iconSettings ~= ipolicy;
}
} else {
// no explicit icon policy was given, so we use a default policy
iconSettings.reserve (6);
iconSettings ~= IconPolicy (ImageSize (48), true, false);
iconSettings ~= IconPolicy (ImageSize (48, 48, 2), true, false);
iconSettings ~= IconPolicy (ImageSize (64), true, false);
iconSettings ~= IconPolicy (ImageSize (64, 64, 2), true, false);
iconSettings ~= IconPolicy (ImageSize (128), true, true);
iconSettings ~= IconPolicy (ImageSize (128, 128, 2), true, true);
}
this.maxScrFileSize = 14; // 14MiB is the default maximum size
if ("MaxScreenshotFileSize" in root)
this.maxScrFileSize = root["MaxScreenshotFileSize"].integer;
if ("AllowedCustomKeys" in root.object)
foreach (ref key; root["AllowedCustomKeys"].array)
allowedCustomKeys[key.str] = true;
// Enable features which are default-enabled
feature.processDesktop = true;
feature.validate = true;
feature.storeScreenshots = true;
feature.optipng = true;
feature.metadataTimestamps = true;
feature.immutableSuites = true;
feature.processFonts = true;
feature.allowIconUpscale = true;
feature.processGStreamer = true;
feature.processLocale = true;
feature.screenshotVideos = true;
// apply vendor feature settings
if ("Features" in root.object) {
auto featuresObj = root["Features"].object;
foreach (featureId; featuresObj.byKey ()) {
switch (featureId) {
case "validateMetainfo":
feature.validate = featuresObj[featureId].type == JSONType.true_;
break;
case "processDesktop":
feature.processDesktop = featuresObj[featureId].type == JSONType.true_;
break;
case "noDownloads":
feature.noDownloads = featuresObj[featureId].type == JSONType.true_;
break;
case "createScreenshotsStore":
feature.storeScreenshots = featuresObj[featureId].type == JSONType.true_;
break;
case "optimizePNGSize":
feature.optipng = featuresObj[featureId].type == JSONType.true_;
break;
case "metadataTimestamps":
feature.metadataTimestamps = featuresObj[featureId].type == JSONType.true_;
break;
case "immutableSuites":
feature.immutableSuites = featuresObj[featureId].type == JSONType.true_;
break;
case "processFonts":
feature.processFonts = featuresObj[featureId].type == JSONType.true_;
break;
case "allowIconUpscaling":
feature.allowIconUpscale = featuresObj[featureId].type == JSONType.true_;
break;
case "processGStreamer":
feature.processGStreamer = featuresObj[featureId].type == JSONType.true_;
break;
case "processLocale":
feature.processLocale = featuresObj[featureId].type == JSONType.true_;
break;
case "screenshotVideos":
feature.screenshotVideos = featuresObj[featureId].type == JSONType.true_;
break;
case "propagateMetaInfoArtifacts":
feature.propagateMetaInfoArtifacts = featuresObj[featureId].type == JSONType.true_;
break;
default:
break;
}
}
}
// check if we need to disable features because some prerequisites are not met
if (feature.optipng) {
if (optipngBinary.empty) {
feature.optipng = false;
logError ("Disabled feature `optimizePNGSize`: The `optipng` binary was not found.");
} else {
logDebug ("Using `optipng`: %s", optipngBinary);
}
}
Globals.setUseOptipng (feature.optipng);
if (feature.screenshotVideos) {
if (ffprobeBinary.empty) {
feature.screenshotVideos = false;
logError ("Disabled feature `screenshotVideos`: The `ffprobe` binary was not found.");
} else {
logDebug ("Using `ffprobe`: %s", ffprobeBinary);
}
}
if (feature.noDownloads) {
// since disallowing network access might have quite a lot of sideeffects, we print
// a message to the logs to make debugging easier.
// in general, running with noDownloads is discouraged.
logWarning ("Configuration does not permit downloading files. Several features will not be available.");
}
if (!feature.immutableSuites) {
// Immutable suites won't work if the feature is disabled - log this error
if (hasImmutableSuites)
logError ("Suites are defined as immutable, but the `immutableSuites` feature is disabled. Immutability will not work!");
}
if (!feature.validate)
logWarning ("MetaInfo validation has been disabled in configuration.");
// sanity check to warn if our GdkPixbuf does not support the minimum amount
// of image formats we need
import ascompose.Image : Image;
import std.string : toStringz;
auto pbFormatNames = Image.supportedFormatNames ();
if (!pbFormatNames.contains(cast(char*) "png".toStringz) ||
!pbFormatNames.contains(cast(char*) "svg".toStringz) ||
!pbFormatNames.contains(cast(char*) "jpeg".toStringz)) {
logError ("The currently used GdkPixbuf does not seem to support all image formats we require to run normally (png/svg/jpeg). " ~
"This may be a problem with your installation of appstream-generator or gdk-pixbuf.");
}
}
bool isValid ()
{
return this.projectName != null;
}
/**
* Get unique temporary directory to use during one generator run.
*/
string getTmpDir ()
{
synchronized (this) {
if (tmpDir.empty) {
string root;
if (cacheRootDir.empty)
root = "/tmp/";
else
root = cacheRootDir;
tmpDir = buildPath (root, "tmp", format ("asgen-%s", randomString (8)));
// make appstream-compose internal functions aware of the new temp dir
Globals.setTmpDir (tmpDir);
}
}
return tmpDir;
}
}
appstream-generator-0.8.6/src/asgen/contentsstore.d 0000664 0000000 0000000 00000027562 14160667064 0022476 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2021 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.contentsstore;
import std.stdio;
import std.string;
import std.conv : to, octal;
import std.file : mkdirRecurse;
import std.array : appender, join, split, empty;
import asgen.bindings.lmdb;
import asgen.config;
import asgen.logging;
/**
* Contains a cache about available files in packages.
* This is useful for finding icons and for re-scanning
* packages which become interesting later.
**/
final class ContentsStore
{
private:
MDB_envp dbEnv;
MDB_dbi dbContents;
MDB_dbi dbIcons;
MDB_dbi dbLocale;
bool opened;
public:
this ()
{
opened = false;
}
~this ()
{
close ();
}
private void checkError (int rc, string msg)
{
if (rc != 0) {
import std.format;
throw new Exception (format ("%s[%s]: %s", msg, rc, mdb_strerror (rc).fromStringz));
}
}
void open (string dir)
{
static import std.math;
int rc;
assert (!opened);
logDebug ("Opening contents cache.");
// ensure the cache directory exists
mkdirRecurse (dir);
rc = mdb_env_create (&dbEnv);
scope (success) opened = true;
scope (failure) dbEnv.mdb_env_close ();
checkError (rc, "mdb_env_create");
// We are going to use at max 3 sub-databases:
// contents, icons and locale
rc = dbEnv.mdb_env_set_maxdbs (3);
checkError (rc, "mdb_env_set_maxdbs");
// set a huge map size to be futureproof.
// This means we're cruel to non-64bit users, but this
// software is supposed to be run on 64bit machines anyway.
auto mapsize = cast (size_t) std.math.pow (512L, 4);
rc = dbEnv.mdb_env_set_mapsize (mapsize);
checkError (rc, "mdb_env_set_mapsize");
// open database
rc = dbEnv.mdb_env_open (dir.toStringz (),
MDB_NOMETASYNC,
octal!755);
checkError (rc, "mdb_env_open");
// open sub-databases in the environment
MDB_txnp txn;
rc = dbEnv.mdb_txn_begin (null, 0, &txn);
checkError (rc, "mdb_txn_begin");
scope (failure) txn.mdb_txn_abort ();
// contains a full list of all contents
rc = txn.mdb_dbi_open ("contents", MDB_CREATE, &dbContents);
checkError (rc, "open contents database");
// contains list of icon files and related data
// the contents sub-database exists only to allow building instances
// of IconHandler much faster.
rc = txn.mdb_dbi_open ("icondata", MDB_CREATE, &dbIcons);
checkError (rc, "open icon-info database");
// contains list of locale files and related data
rc = txn.mdb_dbi_open ("localedata", MDB_CREATE, &dbLocale);
checkError (rc, "open locale-info database");
rc = txn.mdb_txn_commit ();
checkError (rc, "mdb_txn_commit");
}
void open (Config conf)
{
import std.path : buildPath;
this.open (buildPath (conf.databaseDir, "contents"));
}
void close ()
{
synchronized (this) {
if (opened)
dbEnv.mdb_env_close ();
opened = false;
dbEnv = null;
}
}
private MDB_val makeDbValue (const string data)
{
import core.stdc.string : strlen;
MDB_val mval;
auto cdata = data.toStringz ();
mval.mv_size = char.sizeof * strlen (cdata) + 1;
mval.mv_data = cast(void*) cdata;
return mval;
}
private MDB_txnp newTransaction (uint flags = 0)
in { assert (opened); }
do
{
int rc;
MDB_txnp txn;
rc = dbEnv.mdb_txn_begin (null, flags, &txn);
checkError (rc, "mdb_txn_begin");
return txn;
}
private void commitTransaction (MDB_txnp txn)
{
auto rc = txn.mdb_txn_commit ();
checkError (rc, "mdb_txn_commit");
}
private void quitTransaction (MDB_txnp txn)
{
if (txn is null)
return;
txn.mdb_txn_abort ();
}
/**
* Drop a package-id from the contents cache.
*/
void removePackage (string pkid)
{
MDB_val key;
key = makeDbValue (pkid);
auto txn = newTransaction ();
scope (success) commitTransaction (txn);
scope (failure) quitTransaction (txn);
auto res = txn.mdb_del (dbContents, &key, null);
checkError (res, "mdb_del (contents)");
res = txn.mdb_del (dbIcons, &key, null);
if (res != MDB_NOTFOUND)
checkError (res, "mdb_del (icons)");
res = txn.mdb_del (dbLocale, &key, null);
if (res != MDB_NOTFOUND)
checkError (res, "mdb_del (locale)");
}
bool packageExists (const string pkid)
{
MDB_val dkey;
MDB_cursorp cur;
dkey = makeDbValue (pkid);
auto txn = newTransaction (MDB_RDONLY);
scope (exit) quitTransaction (txn);
auto res = txn.mdb_cursor_open (dbContents, &cur);
scope (exit) cur.mdb_cursor_close ();
checkError (res, "mdb_cursor_open");
res = cur.mdb_cursor_get (&dkey, null, MDB_SET);
if (res == MDB_NOTFOUND)
return false;
checkError (res, "mdb_cursor_get");
return true;
}
void addContents (string pkid, string[] contents)
{
// filter out icon filenames and filenames of icon-related stuff (e.g. theme.index),
// as well as locale information
auto iconInfo = appender!(string[]);
auto localeInfo = appender!(string[]);
foreach (ref f; contents) {
if ((f.startsWith ("/usr/share/icons/")) ||
(f.startsWith ("/usr/share/pixmaps/"))) {
iconInfo ~= f;
continue;
}
// create a huge index of all Gettext and Qt translation filenames
if (f.endsWith (".mo") || f.endsWith (".qm")) {
localeInfo ~= f;
continue;
}
}
immutable contentsStr = contents.join ("\n");
synchronized (this) {
MDB_val key, contentsVal;
key = makeDbValue (pkid);
contentsVal = makeDbValue (contentsStr);
auto txn = newTransaction ();
scope (success) commitTransaction (txn);
scope (failure) quitTransaction (txn);
auto res = txn.mdb_put (dbContents, &key, &contentsVal, 0);
checkError (res, "mdb_put");
// if we have icon information, store that too
if (!iconInfo.data.empty) {
MDB_val iconsVal;
immutable iconsStr = iconInfo.data.join ("\n");
iconsVal = makeDbValue (iconsStr);
res = txn.mdb_put (dbIcons, &key, &iconsVal, 0);
checkError (res, "mdb_put (icons)");
}
// store locale
if (!localeInfo.data.empty) {
MDB_val localeVal;
immutable localeStr = localeInfo.data.join ("\n");
localeVal = makeDbValue (localeStr);
res = txn.mdb_put (dbLocale, &key, &localeVal, 0);
checkError (res, "mdb_put (locale)");
}
}
}
private string[string] getFilesMap (string[] pkids, MDB_dbi dbi, bool useBaseName = false)
{
import std.path : baseName;
MDB_cursorp cur;
auto txn = newTransaction (MDB_RDONLY);
scope (exit) quitTransaction (txn);
auto res = txn.mdb_cursor_open (dbi, &cur);
scope (exit) cur.mdb_cursor_close ();
checkError (res, "mdb_cursor_open");
string[string] pkgCMap;
foreach (ref pkid; pkids) {
MDB_val pkey = makeDbValue (pkid);
MDB_val cval;
res = cur.mdb_cursor_get (&pkey, &cval, MDB_SET);
if (res == MDB_NOTFOUND)
continue;
checkError (res, "mdb_cursor_get");
auto data = fromStringz (cast(char*) cval.mv_data);
auto contents = to!string (data);
foreach (const ref c; contents.split ("\n")) {
if (useBaseName)
pkgCMap[c.baseName] = pkid;
else
pkgCMap[c] = pkid;
}
}
return pkgCMap;
}
auto getContentsMap (string[] pkids)
{
return getFilesMap (pkids, dbContents);
}
auto getIconFilesMap (string[] pkids)
{
return getFilesMap (pkids, dbIcons);
}
auto getLocaleMap (string[] pkids)
{
// we make the assumption here that all locale for a given domain are in one package.
// otherwise this global search will get even more insane.
return getFilesMap (pkids, dbLocale);
}
private string[] getContentsList (string pkid, MDB_dbi dbi)
{
MDB_val pkey, cval;
MDB_cursorp cur;
pkey = makeDbValue (pkid);
auto txn = newTransaction (MDB_RDONLY);
scope (exit) quitTransaction (txn);
auto res = txn.mdb_cursor_open (dbi, &cur);
scope (exit) cur.mdb_cursor_close ();
checkError (res, "mdb_cursor_open");
res = cur.mdb_cursor_get (&pkey, &cval, MDB_SET);
if (res == MDB_NOTFOUND)
return null;
checkError (res, "mdb_cursor_get");
auto data = fromStringz (cast(char*) cval.mv_data);
auto contentsStr = to!string (data);
return contentsStr.split ("\n");
}
string[] getContents (string pkid)
{
return getContentsList (pkid, dbContents);
}
string[] getIcons (string pkid)
{
return getContentsList (pkid, dbIcons);
}
string[] getLocaleFiles (string pkid)
{
return getContentsList (pkid, dbLocale);
}
bool[immutable string] getPackageIdSet ()
{
MDB_cursorp cur;
auto txn = newTransaction ();
scope (exit) quitTransaction (txn);
auto res = txn.mdb_cursor_open (dbContents, &cur);
scope (exit) cur.mdb_cursor_close ();
checkError (res, "mdb_cursor_open (getPackageIdSet)");
bool[immutable string] pkgSet;
MDB_val pkey;
while (cur.mdb_cursor_get (&pkey, null, MDB_NEXT) == 0) {
immutable pkid = to!string (fromStringz (cast(char*) pkey.mv_data));
pkgSet[pkid] = true;
}
return pkgSet;
}
void removePackages (ref bool[immutable string] pkidSet)
{
auto txn = newTransaction ();
scope (success) commitTransaction (txn);
scope (failure) quitTransaction (txn);
foreach (ref pkid; pkidSet.byKey) {
auto key = makeDbValue (pkid);
auto res = txn.mdb_del (dbContents, &key, null);
checkError (res, "mdb_del (contents)");
res = txn.mdb_del (dbIcons, &key, null);
if (res != MDB_NOTFOUND)
checkError (res, "mdb_del (icons)");
res = txn.mdb_del (dbLocale, &key, null);
if (res != MDB_NOTFOUND)
checkError (res, "mdb_del (locale)");
}
}
void sync ()
in { assert (opened); }
do
{
dbEnv.mdb_env_sync (1);
}
}
appstream-generator-0.8.6/src/asgen/datainjectpkg.d 0000664 0000000 0000000 00000010747 14160667064 0022371 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2018-2021 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
import std.stdio : File;
import std.array : empty;
import std.path : buildPath, buildNormalizedPath, relativePath, baseName;
import asgen.backends.interfaces;
import asgen.utils : GENERIC_BUFFER_SIZE, existsAndIsDir;
import asgen.logging;
/**
* Fake package which has the sole purpose of allowing easy injection of local
* data that does not reside in packages.
*/
final class DataInjectPackage : Package
{
private:
string pkgname;
string pkgarch;
string pkgmaintainer;
string[string] desc;
string[string] _contents;
string _dataLocation;
string _archDataLocation;
public:
@property override string name () const { return pkgname; }
@property override string ver () const { return "0~0"; }
@property override string arch () const { return pkgarch; }
@property override PackageKind kind () @safe pure { return PackageKind.FAKE; }
@property override const(string[string]) description () const { return desc; }
override
@property string getFilename () const { return "_local_"; }
override
@property string maintainer () const { return pkgmaintainer; }
@property void maintainer (string maint) { pkgmaintainer = maint; }
@property string dataLocation () const { return _dataLocation; }
@property void dataLocation (string value) { _dataLocation = value; }
@property string archDataLocation () const { return _archDataLocation; }
@property void archDataLocation (string value) { _archDataLocation = value; }
this (string pname, string parch)
{
pkgname = pname;
pkgarch = parch;
}
override
ubyte[] getFileData (string fname)
{
immutable localPath = _contents.get (fname, null);
if (localPath.empty)
return [];
ubyte[] data;
auto f = File (localPath, "r");
while (!f.eof) {
char[GENERIC_BUFFER_SIZE] buf;
data ~= f.rawRead (buf);
}
return data;
}
@property override
string[] contents ()
{
import std.file : dirEntries, SpanMode;
import std.array : array;
if (!_contents.empty)
return array(_contents.byKey);
if (_dataLocation.empty || !_dataLocation.existsAndIsDir)
return [];
// find all icons
immutable iconLocation = buildNormalizedPath (_dataLocation, "icons");
foreach (iconFname; iconLocation.dirEntries ("*.{svg,svgz,png}", SpanMode.breadth, true)) {
immutable iconBasePath = relativePath (iconFname, iconLocation);
_contents[buildPath ("/usr/share/icons/hicolor", iconBasePath)] = iconFname;
}
// find metainfo files
foreach (miFname; _dataLocation.dirEntries ("*.xml", SpanMode.shallow, false)) {
immutable miBasename = miFname.baseName;
logDebug ("Found injected metainfo [%s]: %s", "all", miBasename);
_contents[buildPath ("/usr/share/metainfo", miBasename)] = miFname;
}
if (!archDataLocation.existsAndIsDir)
return array(_contents.byKey);
// load arch-specific override metainfo files
foreach (miFname; archDataLocation.dirEntries ("*.xml", SpanMode.shallow, false)) {
immutable miBasename = miFname.baseName;
immutable fakePath = buildPath ("/usr/share/metainfo", miBasename);
if (fakePath in _contents)
logDebug ("Found injected metainfo [%s]: %s (replacing generic one)", arch, miBasename);
else
logDebug ("Found injected metainfo [%s]: %s", arch, miBasename);
_contents[fakePath] = miFname;
}
return array(_contents.byKey);
}
override
void finish ()
{
}
}
appstream-generator-0.8.6/src/asgen/datastore.d 0000664 0000000 0000000 00000054373 14160667064 0021552 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2021 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.datastore;
import std.stdio;
import std.string;
import std.conv : to, octal;
import std.file : mkdirRecurse;
import std.path : buildPath, buildNormalizedPath, pathSplitter;
import std.array : appender;
import std.typecons : Tuple, scoped;
import std.json;
import asgen.bindings.lmdb;
import appstream.Metadata;
import appstream.Component;
import asgen.config;
import asgen.logging;
import asgen.config : DataType;
import asgen.result;
/**
* Main database containing information about scanned packages,
* the components they provide, the component metadata itself,
* issues found as well as statistics about the metadata evolution
* over time.
**/
final class DataStore
{
private:
MDB_envp dbEnv;
MDB_dbi dbRepoInfo;
MDB_dbi dbPackages;
MDB_dbi dbDataXml;
MDB_dbi dbDataYaml;
MDB_dbi dbHints;
MDB_dbi dbStats;
bool opened;
Metadata mdata;
string mediaDir;
public:
struct StatisticsEntry
{
size_t time;
JSONValue data;
}
this ()
{
opened = false;
mdata = new Metadata ();
mdata.setLocale ("ALL");
mdata.setFormatVersion (Config.get ().formatVersion);
mdata.setWriteHeader(false);
}
~this ()
{
close ();
}
@property
string mediaExportPoolDir ()
{
return mediaDir;
}
private void checkError (int rc, string msg)
{
if (rc != 0) {
import std.format;
throw new Exception (format ("%s[%s]: %s", msg, rc, mdb_strerror (rc).fromStringz));
}
}
private void printVersionDbg ()
{
import std.stdio : writeln;
int major, minor, patch;
auto ver = mdb_version (&major, &minor, &patch);
logDebug ("Using %s major=%s minor=%s patch=%s", ver.fromStringz, major, minor, patch);
}
void open (string dir, string mediaBaseDir)
{
static import std.math;
int rc;
assert (!opened);
// add LMDB version we are using to the debug output
printVersionDbg ();
// ensure the cache directory exists
mkdirRecurse (dir);
rc = mdb_env_create (&dbEnv);
scope (success) opened = true;
scope (failure) dbEnv.mdb_env_close ();
checkError (rc, "mdb_env_create");
// We are going to use at max 6 sub-databases:
// packages, hints, metadata_xml, metadata_yaml, statistics
rc = dbEnv.mdb_env_set_maxdbs (6);
checkError (rc, "mdb_env_set_maxdbs");
// set a huge map size to be futureproof.
// This means we're cruel to non-64bit users, but this
// software is supposed to be run on 64bit machines anyway.
auto mapsize = cast (size_t) std.math.pow (512L, 4);
rc = dbEnv.mdb_env_set_mapsize (mapsize);
checkError (rc, "mdb_env_set_mapsize");
// open database
rc = dbEnv.mdb_env_open (dir.toStringz (),
MDB_NOMETASYNC,
octal!755);
checkError (rc, "mdb_env_open");
// open sub-databases in the environment
MDB_txnp txn;
rc = dbEnv.mdb_txn_begin (null, 0, &txn);
checkError (rc, "mdb_txn_begin");
scope (failure) txn.mdb_txn_abort ();
rc = txn.mdb_dbi_open ("packages", MDB_CREATE, &dbPackages);
checkError (rc, "open packages database");
rc = txn.mdb_dbi_open ("repository", MDB_CREATE, &dbRepoInfo);
checkError (rc, "open repository database");
rc = txn.mdb_dbi_open ("metadata_xml", MDB_CREATE, &dbDataXml);
checkError (rc, "open metadata (xml) database");
rc = txn.mdb_dbi_open ("metadata_yaml", MDB_CREATE, &dbDataYaml);
checkError (rc, "open metadata (yaml) database");
rc = txn.mdb_dbi_open ("hints", MDB_CREATE, &dbHints);
checkError (rc, "open hints database");
rc = txn.mdb_dbi_open ("statistics", MDB_CREATE | MDB_INTEGERKEY, &dbStats);
checkError (rc, "open statistics database");
rc = txn.mdb_txn_commit ();
checkError (rc, "mdb_txn_commit");
this.mediaDir = buildPath (mediaBaseDir, "pool");
mkdirRecurse (this.mediaDir);
}
void close ()
{
synchronized (this) {
if (opened)
dbEnv.mdb_env_close ();
opened = false;
dbEnv = null;
}
}
void open (Config conf)
{
this.open (buildPath (conf.databaseDir, "main"), conf.mediaExportDir);
}
private MDB_val makeDbValue (string data)
{
import core.stdc.string : strlen;
MDB_val mval;
auto cdata = data.toStringz ();
mval.mv_size = char.sizeof * strlen (cdata) + 1;
mval.mv_data = cast(void*) cdata;
return mval;
}
private MDB_txnp newTransaction (uint flags = 0)
in { assert (opened); }
do
{
int rc;
MDB_txnp txn;
rc = dbEnv.mdb_txn_begin (null, flags, &txn);
checkError (rc, "mdb_txn_begin");
return txn;
}
private void commitTransaction (MDB_txnp txn)
{
auto rc = txn.mdb_txn_commit ();
checkError (rc, "mdb_txn_commit");
}
private void quitTransaction (MDB_txnp txn)
{
if (txn is null)
return;
txn.mdb_txn_abort ();
}
private void putKeyValue (MDB_dbi dbi, string key, string value)
{
MDB_val dbkey, dbvalue;
dbkey = makeDbValue (key);
dbvalue = makeDbValue (value);
auto txn = newTransaction ();
scope (success) commitTransaction (txn);
scope (failure) quitTransaction (txn);
auto res = txn.mdb_put (dbi, &dbkey, &dbvalue, 0);
checkError (res, "mdb_put");
}
private string getValue (MDB_dbi dbi, MDB_val dkey)
{
import std.conv;
MDB_val dval;
MDB_cursorp cur;
auto txn = newTransaction (MDB_RDONLY);
scope (exit) quitTransaction (txn);
auto res = txn.mdb_cursor_open (dbi, &cur);
scope (exit) cur.mdb_cursor_close ();
checkError (res, "mdb_cursor_open");
res = cur.mdb_cursor_get (&dkey, &dval, MDB_SET);
if (res == MDB_NOTFOUND)
return null;
checkError (res, "mdb_cursor_get");
auto data = fromStringz (cast(char*) dval.mv_data);
return to!string (data);
}
private string getValue (MDB_dbi dbi, string key)
{
MDB_val dkey;
dkey = makeDbValue (key);
return getValue (dbi, dkey);
}
bool metadataExists (DataType dtype, string gcid)
{
return getMetadata (dtype, gcid) !is null;
}
void setMetadata (DataType dtype, string gcid, string asdata)
{
if (dtype == DataType.XML)
putKeyValue (dbDataXml, gcid, asdata);
else
putKeyValue (dbDataYaml, gcid, asdata);
}
string getMetadata (DataType dtype, string gcid)
{
string data;
if (dtype == DataType.XML)
data = getValue (dbDataXml, gcid);
else
data = getValue (dbDataYaml, gcid);
return data;
}
bool hasHints (string pkid)
{
return getValue (dbHints, pkid) !is null;
}
void setHints (string pkid, string hintsYaml)
{
putKeyValue (dbHints, pkid, hintsYaml);
}
string getHints (string pkid)
{
return getValue (dbHints, pkid);
}
string getPackageValue (string pkid)
{
return getValue (dbPackages, pkid);
}
void setPackageIgnore (string pkid)
{
putKeyValue (dbPackages, pkid, "ignore");
}
bool isIgnored (string pkid)
{
auto val = getValue (dbPackages, pkid);
return val == "ignore";
}
bool packageExists (string pkid)
{
auto val = getValue (dbPackages, pkid);
return val !is null;
}
void addGeneratorResult (DataType dtype, ref GeneratorResult gres, const bool alwaysRegenerate = false)
{
// if the package has no components or hints,
// mark it as always-ignore
if (gres.unitIgnored) {
setPackageIgnore (gres.pkid);
return;
}
auto cptsPtrArray = gres.fetchComponents ();
for (uint i = 0; i < cptsPtrArray.len; i++) {
auto cpt = new Component (cast (AsComponent*) cptsPtrArray.index (i));
auto gcid = gres.gcidForComponent (cpt);
if (metadataExists (dtype, gcid) && !alwaysRegenerate) {
// we already have seen this exact metadata - only adjust the reference,
// and don't regenerate it.
continue;
}
mdata.clearComponents ();
mdata.addComponent (cpt);
// convert out compoent into metadata
string data;
try {
if (dtype == DataType.XML) {
data = mdata.componentsToCollection (FormatKind.XML);
} else {
data = mdata.componentsToCollection (FormatKind.YAML);
}
} catch (Exception e) {
gres.addHint (cpt, "metadata-serialization-failed", e.msg);
continue;
}
// remove trailing whitespaces and linebreaks
data = data.stripRight ();
// store metadata
if (!empty (data))
setMetadata (dtype, gcid, data);
}
if (gres.hintsCount () > 0) {
auto hintsJson = gres.hintsToJson ();
if (!hintsJson.empty)
setHints (gres.pkid, hintsJson);
}
auto gcids = gres.getComponentGcids ();
if (gcids.empty) {
// no global components, and we're not ignoring this component.
// this means we likely have hints stored for this one. Mark it
// as "seen" so we don't reprocess it again.
putKeyValue (dbPackages, gres.pkid, "seen");
} else {
import std.array : join;
// store global component IDs for this package as newline-separated list
auto gcidVal = join (gcids, "\n");
putKeyValue (dbPackages, gres.pkid, gcidVal);
}
}
string[] getGCIDsForPackage (string pkid)
{
auto pkval = getPackageValue (pkid);
if (pkval == "ignore")
return null;
if (pkval == "seen")
return null;
auto validCids = appender!(string[]);
auto cids = pkval.split ("\n");
foreach (cid; cids) {
if (cid.empty)
continue;
validCids ~= cid;
}
return validCids.data;
}
string[] getMetadataForPackage (DataType dtype, string pkid)
{
auto gcids = getGCIDsForPackage (pkid);
if (gcids is null)
return null;
auto res = appender!(string[]);
res.reserve (gcids.length);
foreach (cid; gcids) {
auto data = getMetadata (dtype, cid);
if (!data.empty)
res ~= data;
}
return res.data;
}
/**
* Drop a package from the database. This process might leave cruft behind,
* which can be collected using the cleanupCruft() method.
*/
void removePackage (string pkid)
{
MDB_val dbkey;
dbkey = makeDbValue (pkid);
auto txn = newTransaction ();
scope (success) commitTransaction (txn);
scope (failure) quitTransaction (txn);
auto res = txn.mdb_del (dbPackages, &dbkey, null);
if (res != MDB_NOTFOUND)
checkError (res, "mdb_del");
res = txn.mdb_del (dbHints, &dbkey, null);
if (res != MDB_NOTFOUND)
checkError (res, "mdb_del");
}
private auto getActiveGCIDs ()
{
MDB_val dkey, dval;
MDB_cursorp cur;
auto txn = newTransaction (MDB_RDONLY);
scope (exit) quitTransaction (txn);
auto res = txn.mdb_cursor_open (dbPackages, &cur);
scope (exit) cur.mdb_cursor_close ();
checkError (res, "mdb_cursor_open (gcids)");
bool[string] gcids;
while (cur.mdb_cursor_get (&dkey, &dval, MDB_NEXT) == 0) {
immutable pkval = to!string (fromStringz (cast(char*) dval.mv_data));
if ((pkval == "ignore") || (pkval == "seen"))
continue;
foreach (gcid; pkval.split ("\n"))
gcids[gcid] = true;
}
return gcids;
}
void cleanupCruft ()
{
import std.file;
import std.array : array;
if (mediaDir is null) {
logError ("Can not clean up cruft: No media directory is set.");
return;
}
auto activeGCIDs = getActiveGCIDs ();
bool gcidReferenced (string gcid)
{
// we use an associative array as a set here
return (gcid in activeGCIDs) !is null;
}
void dropOrphanedData (MDB_dbi dbi)
{
MDB_cursorp cur;
auto txn = newTransaction ();
scope (success) commitTransaction (txn);
scope (failure) quitTransaction (txn);
auto res = txn.mdb_cursor_open (dbi, &cur);
scope (exit) cur.mdb_cursor_close ();
checkError (res, "mdb_cursor_open (stats)");
MDB_val ckey;
while (cur.mdb_cursor_get (&ckey, null, MDB_NEXT) == 0) {
immutable gcid = to!string (fromStringz (cast(char*) ckey.mv_data));
if (gcidReferenced (gcid))
continue;
// if we got here, the component is cruft and can be removed
res = cur.mdb_cursor_del (0);
checkError (res, "mdb_del");
logInfo ("Marked %s as cruft.", gcid);
}
}
bool dirEmpty (string dir)
{
bool empty = true;
foreach (ref e; dirEntries (dir, SpanMode.shallow, false)) {
empty = false;
break;
}
return empty;
}
void cleanupDirs (string rootPath) {
auto pdir = buildNormalizedPath (rootPath, "..");
if (!std.file.exists (pdir))
return;
if (dirEmpty (pdir))
rmdir (pdir);
pdir = buildNormalizedPath (pdir, "..");
if (dirEmpty (pdir))
rmdir (pdir);
}
// drop orphaned metadata
dropOrphanedData (dbDataXml);
dropOrphanedData (dbDataYaml);
// we need the global Config instance here
auto conf = Config.get ();
auto mdirLen = mediaDir.length;
foreach (ref path; dirEntries (mediaDir, SpanMode.depth, false)) {
if (path.length <= mdirLen)
continue;
immutable relPath = path[mdirLen+1..$];
auto split = array (pathSplitter (relPath));
if (split.length != 4)
continue;
immutable gcid = relPath;
if (gcidReferenced (gcid))
continue;
// if we are here, the component is removed and we can drop its media
if (std.file.exists (path))
rmdirRecurse (path);
// remove possibly empty directories
cleanupDirs (path);
// expire data in suite-specific media directories,
// if suite is not marked as immutable
if (conf.feature.immutableSuites) {
foreach (ref suite; conf.suites) {
if (suite.isImmutable)
continue;
immutable suiteGCIDMediaDir = buildNormalizedPath (mediaDir, "..", suite.name, gcid);
if (std.file.exists (suiteGCIDMediaDir))
rmdirRecurse (suiteGCIDMediaDir);
// remove possibly empty directories
cleanupDirs (suiteGCIDMediaDir);
}
}
logInfo ("Expired media for '%s'", gcid);
}
}
bool[immutable string] getPackageIdSet ()
{
MDB_cursorp cur;
auto txn = newTransaction ();
scope (exit) quitTransaction (txn);
bool[immutable string] pkgSet;
auto res = txn.mdb_cursor_open (dbPackages, &cur);
scope (exit) cur.mdb_cursor_close ();
checkError (res, "mdb_cursor_open (getPackageIdSet)");
MDB_val pkey;
while (cur.mdb_cursor_get (&pkey, null, MDB_NEXT) == 0) {
immutable pkid = to!string (fromStringz (cast(char*) pkey.mv_data));
pkgSet[pkid] = true;
}
return pkgSet;
}
void removePackages (ref bool[immutable string] pkidSet)
{
auto txn = newTransaction ();
scope (success) commitTransaction (txn);
scope (failure) quitTransaction (txn);
foreach (ref pkid; pkidSet.byKey) {
auto dbkey = makeDbValue (pkid);
auto res = txn.mdb_del (dbPackages, &dbkey, null);
if (res != MDB_NOTFOUND)
checkError (res, "mdb_del (metadata)");
res = txn.mdb_del (dbHints, &dbkey, null);
if (res != MDB_NOTFOUND)
checkError (res, "mdb_del (hints)");
logInfo ("Dropped package %s", pkid);
}
}
StatisticsEntry[] getStatistics ()
{
MDB_val dkey, dval;
MDB_cursorp cur;
auto txn = newTransaction (MDB_RDONLY);
scope (exit) quitTransaction (txn);
auto res = txn.mdb_cursor_open (dbStats, &cur);
scope (exit) cur.mdb_cursor_close ();
checkError (res, "mdb_cursor_open (stats)");
auto stats = appender!(StatisticsEntry[]);
stats.reserve (128);
while (cur.mdb_cursor_get (&dkey, &dval, MDB_NEXT) == 0) {
auto jsonData = to!string (fromStringz (cast(char*) dval.mv_data));
auto timestamp = *(cast(size_t*) dkey.mv_data);
auto sentry = StatisticsEntry (timestamp, parseJSON (jsonData));
stats ~= sentry;
}
return stats.data;
}
void removeStatistics (size_t time)
{
MDB_val dbkey;
dbkey.mv_size = size_t.sizeof;
dbkey.mv_data = &time;
auto txn = newTransaction ();
scope (success) commitTransaction (txn);
scope (failure) quitTransaction (txn);
auto res = txn.mdb_del (dbStats, &dbkey, null);
if (res != MDB_NOTFOUND)
checkError (res, "mdb_del");
}
void addStatistics (JSONValue stats)
{
import core.stdc.time : time;
MDB_val dbkey, dbvalue;
size_t unixTime = time (null);
auto statsJsonStr = stats.toString ();
dbkey.mv_size = size_t.sizeof;
dbkey.mv_data = &unixTime;
dbvalue = makeDbValue (statsJsonStr);
auto txn = newTransaction ();
scope (success) commitTransaction (txn);
scope (failure) quitTransaction (txn);
auto res = txn.mdb_put (dbStats, &dbkey, &dbvalue, MDB_APPEND);
if (res == MDB_KEYEXIST) {
// this point in time already exists, so we need to extend it with additional data
// retrieve the old statistics data
auto existingJsonData = getValue (dbStats, dbkey);
auto existingJson = parseJSON (existingJsonData);
// make the new JSON a list of the old and the new data, if it isn't one already
JSONValue newJson;
if (existingJson.type == JSONType.array) {
newJson = existingJson;
newJson.array ~= stats;
} else {
newJson = JSONValue ([existingJson, stats]);
}
// build new database value and add it to the db, overriding the old one
statsJsonStr = newJson.toJSON ();
dbvalue = makeDbValue (statsJsonStr);
res = txn.mdb_put (dbStats, &dbkey, &dbvalue, 0);
}
checkError (res, "mdb_put (stats)");
}
JSONValue getRepoInfo (string suite, string section, string arch)
{
auto repoid = "%s-%s-%s".format (suite, section, arch);
auto jsonData = getValue (dbRepoInfo, repoid);
if (jsonData is null) {
JSONValue[string] dummy;
return JSONValue (dummy);
}
return parseJSON (jsonData);
}
void setRepoInfo (string suite, string section, string arch, JSONValue repoInfo)
{
auto repoid = "%s-%s-%s".format (suite, section, arch);
auto jsonData = repoInfo.toJSON ();
putKeyValue (dbRepoInfo, repoid, jsonData);
}
void removeRepoInfo (string suite, string section, string arch)
{
auto repoid = "%s-%s-%s".format (suite, section, arch);
auto dbkey = makeDbValue (repoid);
auto txn = newTransaction ();
scope (success) commitTransaction (txn);
scope (failure) quitTransaction (txn);
auto res = txn.mdb_del (dbRepoInfo, &dbkey, null);
if (res != MDB_NOTFOUND)
checkError (res, "mdb_del");
}
/**
* Get a list of package-ids which match a prefix.
*/
string[] getPkidsMatching (string prefix)
{
MDB_val dkey;
MDB_cursorp cur;
auto txn = newTransaction (MDB_RDONLY);
scope (exit) quitTransaction (txn);
auto res = txn.mdb_cursor_open (dbPackages, &cur);
scope (exit) cur.mdb_cursor_close ();
checkError (res, "mdb_cursor_open (pkid-match)");
auto pkids = appender!(string[]);
prefix ~= "/";
while (cur.mdb_cursor_get (&dkey, null, MDB_NEXT) == 0) {
auto pkid = to!string (fromStringz (cast(char*) dkey.mv_data));
if (pkid.startsWith (prefix))
pkids ~= pkid;
}
return pkids.data;
}
}
appstream-generator-0.8.6/src/asgen/defines.d.in 0000664 0000000 0000000 00000001712 14160667064 0021573 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2017 Canonical Ltd
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.defines;
@safe:
// location of arch-indep data for asgen
public enum DATADIR = "@datadir@";
// current version of the tool
public enum ASGEN_VERSION = "@asgen_version@";
appstream-generator-0.8.6/src/asgen/downloader.d 0000664 0000000 0000000 00000023404 14160667064 0021711 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2019-2021 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.downloader;
@safe:
import std.stdio : File;
import std.typecons : Nullable;
import std.datetime : SysTime, Clock, parseRFC822DateTime, DateTimeException;
import std.array : appender, empty;
import std.path : buildPath, dirName, buildNormalizedPath;
import std.algorithm : startsWith;
import std.format : format;
import std.conv : to;
import asgen.logging;
static import std.file;
import asgen.config : Config;
import asgen.defines : ASGEN_VERSION;
import asgen.utils : isRemote, randomString;
class DownloadException : Exception
{
@safe pure nothrow
this(const string msg,
const string file = __FILE__,
size_t line = __LINE__,
Throwable next = null)
{
super (msg, file, line, next);
}
}
/**
* Download data via HTTP. Based on cURL.
*/
final class Downloader
{
private:
immutable string caInfo;
immutable string userAgent;
// thread local instance
static Downloader instance_;
public:
static Downloader get () @trusted
{
if (instance_ is null)
instance_ = new Downloader;
return instance_;
}
this () @trusted
{
userAgent = "appstream-generator/" ~ ASGEN_VERSION;
// set custom SSL CA file, if we have one
caInfo = Config.get.caInfo;
}
private immutable(Nullable!SysTime) downloadInternal (const string url, ref File dest, const uint maxTryCount = 5) @trusted
in { assert (url.isRemote); }
do
{
import core.time : dur;
import std.string : toLower;
import std.net.curl : HTTP, FTP;
import std.typecons : Yes;
Nullable!SysTime ret;
size_t onReceiveCb (File f, ubyte[] data)
{
f.rawWrite (data);
return data.length;
}
/* the curl library is stupid; you can't make an AutoProtocol set timeouts */
logDebug ("Downloading %s", url);
try {
if (url.startsWith ("http")) {
immutable httpsUrl = url.startsWith ("https");
auto curlHttp = HTTP (url);
if (!caInfo.empty)
curlHttp.caInfo = caInfo;
curlHttp.setUserAgent (userAgent);
HTTP.StatusLine statusLine;
curlHttp.connectTimeout = dur!"seconds" (30);
curlHttp.dataTimeout = dur!"seconds" (30);
curlHttp.onReceive = (data) => onReceiveCb (dest, data);
curlHttp.onReceiveStatusLine = (HTTP.StatusLine l) { statusLine = l; };
curlHttp.onReceiveHeader = (in char[] key, in char[] value) {
// we will not allow a HTTPS --> HTTP downgrade
if (!httpsUrl)
return;
if (key == "location" && value.toLower.startsWith ("http:"))
throw new DownloadException ("HTTPS URL tried to redirect to a less secure HTTP URL.");
};
curlHttp.perform(Yes.throwOnError);
if ("last-modified" in curlHttp.responseHeaders) {
auto lastmodified = curlHttp.responseHeaders["last-modified"];
ret = parseRFC822DateTime(lastmodified);
}
if (statusLine.code != 200 && statusLine.code != 301 && statusLine.code != 302) {
if (statusLine.code == 0) {
// with some recent update of the D runtime or Curl, the status line isn't set anymore
// just to be safe, check whether we received data before assuming everything went fine
if (dest.size == 0)
throw new DownloadException ("No data was received from the remote end (Code: %d).".format (statusLine.code));
} else {
throw new DownloadException ("HTTP request returned status code %d (%s)".format (statusLine.code, statusLine.reason));
}
}
} else {
auto curlFtp = FTP (url);
curlFtp.connectTimeout = dur!"seconds" (30);
curlFtp.dataTimeout = dur!"seconds" (30);
curlFtp.onReceive = (data) => onReceiveCb (dest, data);
curlFtp.perform(Yes.throwOnError);
}
logDebug ("Downloaded %s", url);
} catch (Exception e) {
if (maxTryCount > 0) {
logDebug ("Failed to download %s, will retry %d more %s",
url,
maxTryCount,
maxTryCount > 1 ? "times" : "time");
download (url, dest, maxTryCount - 1);
} else {
throw new DownloadException (e.message.to!string);
}
}
return ret;
}
immutable(Nullable!SysTime) download (const string url, ref File dFile, const uint maxTryCount = 3) @trusted
{
return downloadInternal (url, dFile, maxTryCount);
}
immutable(ubyte[]) download (const string url, const uint maxTryCount = 3) @trusted
{
import core.stdc.stdlib : free;
import core.sys.posix.stdio : fclose, open_memstream;
char* ptr = null;
scope (exit) free (ptr);
size_t sz = 0;
{
auto f = open_memstream (&ptr, &sz);
scope (exit) fclose (f);
auto file = File.wrapFile (f);
downloadInternal (url, file, maxTryCount);
}
return cast(immutable ubyte[]) ptr[0..sz].idup;
}
/**
* Download `url` to `dest`.
*
* Params:
* url = The URL to download.
* dest = The location for the downloaded file.
* maxTryCount = Number of times to attempt the download.
*/
void downloadFile (const string url, const string dest, const uint maxTryCount = 3) @trusted
in { assert (url.isRemote); }
out { assert (std.file.exists (dest)); }
do
{
import std.file : exists, mkdirRecurse, setTimes, remove;
static import std.file;
if (dest.exists) {
logDebug ("File '%s' already exists, re-download of '%s' skipped.", dest, url);
return;
}
mkdirRecurse (dest.dirName);
auto f = File (dest, "wb");
scope (failure) remove (dest);
auto time = downloadInternal (url, f, maxTryCount);
f.close ();
if (!time.isNull)
setTimes (dest, Clock.currTime, time.get);
}
/**
* Download `url` and return a string with its contents.
*
* Params:
* url = The URL to download.
* maxTryCount = Number of times to retry on timeout.
*/
string downloadText (const string url, const uint maxTryCount = 3) @trusted
{
const data = download (url, maxTryCount);
return (cast(char[])data).to!string;
}
/**
* Download `url` and return a string array of lines.
*
* Params:
* url = The URL to download.
* maxTryCount = Number of times to retry on timeout.
*/
string[] downloadTextLines (const string url, const uint maxTryCount = 3) @trusted
{
import std.string : splitLines;
return downloadText (url, maxTryCount).splitLines;
}
}
@trusted
unittest
{
import std.stdio : writeln;
import std.exception : assertThrown;
import std.file : remove, readText;
import std.process : environment;
asgen.logging.setVerbose (true);
writeln ("TEST: ", "Downloader");
if (environment.get("ASGEN_TESTS_NO_NET", "no") != "no") {
writeln ("I: NETWORK DEPENDENT TESTS SKIPPED. (explicitly disabled via `ASGEN_TESTS_NO_NET`)");
return;
}
immutable urlFirefoxDetectportal = "https://detectportal.firefox.com/";
auto dl = new Downloader;
string detectPortalRes;
try {
detectPortalRes = dl.downloadText (urlFirefoxDetectportal);
} catch (DownloadException e) {
writeln ("W: NETWORK DEPENDENT TESTS SKIPPED. (automatically, no network detected: ", e.msg, ")");
return;
}
writeln ("I: Running network-dependent tests.");
assert (detectPortalRes == "success\n");
// check if a downloaded file contains the right contents
immutable firefoxDetectportalFname = "/tmp/asgen-test.ffdp" ~ randomString (4);
scope(exit) firefoxDetectportalFname.remove ();
dl.downloadFile (urlFirefoxDetectportal, firefoxDetectportalFname);
assert (readText (firefoxDetectportalFname) == "success\n");
// download a bigger chunk of data without error
immutable debianOrgFname = "/tmp/asgen-test.do" ~ randomString (4);
scope(exit) debianOrgFname.remove ();
dl.downloadFile ("https://debian.org", debianOrgFname);
// fail when attempting to download a nonexistent file
assertThrown!DownloadException (dl.downloadFile ("https://appstream.debian.org/nonexistent", "/tmp/asgen-dltest" ~ randomString (4), 2));
// check if HTTP --> HTTPS redirects, like done on mozilla.org, work
dl.downloadFile ("http://mozilla.org", "/tmp/asgen-test.mozilla" ~ randomString (4), 1);
}
appstream-generator-0.8.6/src/asgen/engine.d 0000664 0000000 0000000 00000124725 14160667064 0021030 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2021 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.engine;
import std.parallelism : parallel;
import std.string : format, count, toLower, startsWith;
import std.array : Appender, appender, empty;
import std.path : buildPath, buildNormalizedPath;
import std.file : mkdirRecurse, rmdirRecurse;
import std.algorithm : canFind, sort, SwapStrategy;
import std.typecons : scoped, Nullable, Tuple;
import std.conv : to;
static import std.file;
import appstream.Component;
import asgen.config;
import asgen.logging;
import asgen.extractor;
import asgen.datastore;
import asgen.contentsstore;
import asgen.result;
import asgen.hint;
import asgen.reportgenerator;
import asgen.localeunit : LocaleUnit;
import asgen.utils : copyDir, stringArrayToByteArray, getCidFromGlobalID;
import asgen.backends.interfaces;
import asgen.backends.dummy;
import asgen.backends.debian;
import asgen.backends.ubuntu;
import asgen.backends.archlinux;
import asgen.backends.rpmmd;
import asgen.backends.alpinelinux;
import asgen.iconhandler : IconHandler;
/**
* Class orchestrating the whole metadata extraction
* and publication process.
*/
final class Engine
{
private:
Config conf;
PackageIndex pkgIndex;
DataStore dstore;
ContentsStore cstore;
bool m_forced;
public:
this ()
{
this.conf = Config.get ();
switch (conf.backend) {
case Backend.Dummy:
pkgIndex = new DummyPackageIndex (conf.archiveRoot);
break;
case Backend.Debian:
pkgIndex = new DebianPackageIndex (conf.archiveRoot);
break;
case Backend.Ubuntu:
pkgIndex = new UbuntuPackageIndex (conf.archiveRoot);
break;
case Backend.Archlinux:
pkgIndex = new ArchPackageIndex (conf.archiveRoot);
break;
case Backend.RpmMd:
pkgIndex = new RPMPackageIndex (conf.archiveRoot);
break;
case Backend.Alpinelinux:
pkgIndex = new AlpinePackageIndex (conf.archiveRoot);
break;
default:
throw new Exception ("No backend specified, can not continue!");
}
// load global registry of issue hint templates
loadHintsRegistry ();
// create cache in cache directory on workspace
dstore = new DataStore ();
dstore.open (conf);
// open package contents cache
cstore = new ContentsStore ();
cstore.open (conf);
}
@property
bool forced ()
{
return m_forced;
}
@property
void forced (bool v)
{
m_forced = v;
}
private void gcCollect ()
{
static import core.memory;
logDebug ("GC collection cycle triggered explicitly.");
core.memory.GC.collect ();
}
private void logVersionInfo ()
{
import asgen.defines : ASGEN_VERSION;
static import appstream.Utils;
alias AsUtils = appstream.Utils.Utils;
string backendInfo = "";
if (!conf.backendName.empty)
backendInfo = " [%s]".format (conf.backendName);
logInfo ("AppStream Generator %s, AS: %s%s", ASGEN_VERSION, AsUtils.appstreamVersion, backendInfo);
}
/**
* Extract metadata from a software container (usually a distro package).
* The result is automatically stored in the database.
*/
private void processPackages (ref Package[] pkgs, IconHandler iconh)
{
import std.range : chunks;
import glib.Thread : Thread;
auto localeUnit = new LocaleUnit (cstore, pkgs);
ulong chunkSize = pkgs.length / Thread.getNumProcessors () / 10;
if (chunkSize > 100)
chunkSize = 100;
if (chunkSize <= 10)
chunkSize = 10;
logDebug ("Analyzing %s packages in batches of %s", pkgs.length, chunkSize);
foreach (pkgsChunk; parallel (pkgs.chunks (chunkSize), 1)) {
auto mde = new DataExtractor (dstore, iconh, localeUnit);
foreach (ref pkg; pkgsChunk) {
immutable pkid = pkg.id;
if (dstore.packageExists (pkid))
continue;
auto res = mde.processPackage (pkg);
synchronized (dstore) {
// write resulting data into the database
dstore.addGeneratorResult (this.conf.metadataType, res);
logInfo ("Processed %s, components: %s, hints: %s",
res.pkid, res.componentsCount (), res.hintsCount ());
}
// we don't need content data from this package anymore
pkg.finish ();
}
}
}
/**
* Populate the contents index with new contents data. While we are at it, we can also mark
* some uninteresting packages as to-be-ignored, so we don't waste time on them
* during the following metadata extraction.
*
* Returns: True in case we have new interesting packages, false otherwise.
**/
private bool seedContentsData (Suite suite, string section, string arch, Package[] pkgs = [])
{
import glib.Thread : Thread;
bool packageInteresting (Package pkg)
{
auto contents = pkg.contents;
foreach (ref c; contents) {
if (c.startsWith ("/usr/share/applications/"))
return true;
if (c.startsWith ("/usr/share/metainfo/"))
return true;
}
if (pkg.gst.isNull)
return false;
return pkg.gst.get.isNotEmpty;
}
ulong workUnitSize = Thread.getNumProcessors * 2;
if (workUnitSize >= pkgs.length)
workUnitSize = 4;
if (workUnitSize > 30)
workUnitSize = 30;
logDebug ("Scanning %s packages, work unit size %s", pkgs.length, workUnitSize);
// check if the index has changed data, skip the update if there's nothing new
if ((pkgs.empty) && (!pkgIndex.hasChanges (dstore, suite.name, section, arch)) && (!this.forced)) {
logDebug ("Skipping contents cache update for %s/%s [%s], index has not changed.", suite.name, section, arch);
return false;
}
logInfo ("Scanning new packages for %s/%s [%s]", suite.name, section, arch);
if (pkgs.empty)
pkgs = pkgIndex.packagesFor (suite.name, section, arch);
// get contents information for packages and add them to the database
auto interestingFound = false;
// First get the contents (only) of all packages in the base suite
if (!suite.baseSuite.empty) {
logInfo ("Scanning new packages for base suite %s/%s [%s]", suite.baseSuite, section, arch);
auto baseSuitePkgs = pkgIndex.packagesFor (suite.baseSuite, section, arch);
foreach (ref pkg; parallel (baseSuitePkgs, workUnitSize)) {
immutable pkid = pkg.id;
if (!cstore.packageExists (pkid)) {
cstore.addContents (pkid, pkg.contents);
logInfo ("Scanned %s for base suite.", pkid);
}
// chances are that we might never want to extract data from these packages,
// so remove their temporary data for now - we can reopen the packages later if we actually need them.
pkg.cleanupTemp ();
}
}
// And then scan the suite itself - here packages can be 'interesting'
// in that they might end up in the output.
foreach (ref pkg; parallel (pkgs, workUnitSize)) {
immutable pkid = pkg.id;
string[] contents;
if (cstore.packageExists (pkid)) {
if (dstore.packageExists (pkid)) {
// TODO: Unfortunately, packages can move between suites without changing their ID.
// This means as soon as we have an interesting package, even if we already processed it,
// we need to regenerate the output metadata.
// For that to happen, we set interestingFound to true here. Later, a more elegent solution
// would be desirable here, ideally one which doesn't force us to track which package is
// in which suite as well.
if (!dstore.isIgnored (pkid))
interestingFound = true;
continue;
}
// we will complement the main database with ignore data, in case it
// went missing.
contents = cstore.getContents (pkid);
} else {
// add contents to the index
contents = pkg.contents;
cstore.addContents (pkid, contents);
}
// check if we can already mark this package as ignored, and print some log messages
if (!packageInteresting (pkg)) {
dstore.setPackageIgnore (pkid);
logInfo ("Scanned %s, no interesting files found.", pkid);
// we won't use this anymore
pkg.finish ();
} else {
logInfo ("Scanned %s, could be interesting.", pkid);
interestingFound = true;
}
}
// ensure the contents store is in a consistent state on disk,
// since it might be accessed from other threads after this function
// is run.
cstore.sync ();
return interestingFound;
}
private string getMetadataHead (Suite suite, string section)
{
import std.datetime : Clock;
import core.time : Duration;
string head;
immutable origin = "%s-%s-%s".format (conf.projectName.toLower, suite.name.toLower, section.toLower);
auto time = Clock.currTime ();
time.fracSecs = Duration.zero; // we don't want fractional seconds
immutable timeStr = time.toISOString ();
string mediaPoolUrl = buildPath (conf.mediaBaseUrl, "pool");
if (conf.feature.immutableSuites) {
mediaPoolUrl = buildPath (conf.mediaBaseUrl, suite.name);
}
if (conf.metadataType == DataType.XML) {
head = "\n";
head ~= format ("//");
return false;
}
immutable pkid = identifier;
writeln ("== ", pkid, " ==");
writeln ("Contents:");
auto pkgContents = cstore.getContents (pkid);
if (pkgContents.empty) {
writeln ("~ No contents found.");
} else {
foreach (ref s; pkgContents)
writeln (" ", s);
}
writeln ();
writeln ("Icons:");
auto pkgIcons = cstore.getIcons (pkid);
if (pkgIcons.empty) {
writeln ("~ No icons found.");
} else {
foreach (ref s; pkgIcons)
writeln (" ", s);
}
writeln ();
if (dstore.isIgnored (pkid)) {
writeln ("Ignored: yes");
writeln ();
} else {
writeln ("Global Component IDs:");
foreach (ref s; dstore.getGCIDsForPackage (pkid))
writeln ("- ", s);
writeln ();
writeln ("Generated Data:");
foreach (ref s; dstore.getMetadataForPackage (conf.metadataType, pkid))
writeln (s);
writeln ();
}
if (dstore.hasHints (pkid)) {
writeln ("Hints:");
writeln (dstore.getHints (pkid));
} else {
writeln ("Hints: None");
}
writeln ();
return true;
}
} // End of Engine class
appstream-generator-0.8.6/src/asgen/extractor.d 0000664 0000000 0000000 00000033624 14160667064 0021573 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2021 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.extractor;
import std.array : appender;
import std.stdio;
import std.string;
import std.path : baseName;
import std.algorithm : canFind;
import std.typecons : scoped;
import appstream.Component;
import appstream.Metadata;
import ascompose.Hint : Hint;
import ascompose.Compose : Compose, IconPolicy;
import ascompose.Unit : Unit;
import ascompose.c.types : ComposeFlags, AscResult, AscUnit;
import glib.Bytes : Bytes;
import glib.c.types : GPtrArray;
import asgen.config : Config, DataType;
import asgen.logging;
import asgen.hintregistry;
import asgen.result;
import asgen.backends.interfaces;
import asgen.datastore;
import asgen.iconhandler : IconHandler;
import asgen.utils : componentGetRawIcon, toStaticGBytes;
import asgen.packageunit : PackageUnit;
import asgen.localeunit : LocaleUnit;
final class DataExtractor
{
private:
Compose compose;
package DataType dtype;
package DataStore dstore;
package Config conf;
IconHandler iconh;
LocaleUnit l10nUnit;
public:
this (DataStore db, IconHandler iconHandler, LocaleUnit localeUnit)
{
dstore = db;
iconh = iconHandler;
conf = Config.get ();
dtype = conf.metadataType;
l10nUnit = localeUnit;
compose = new Compose;
//compose.setPrefix ("/usr");
compose.setMediaResultDir (db.mediaExportPoolDir);
compose.setMediaBaseurl ("");
compose.setCheckMetadataEarlyFunc (&checkMetadataIntermediate, cast(void*) this);
compose.addFlags (ComposeFlags.IGNORE_ICONS | // we do custom icon processing
ComposeFlags.PROCESS_UNPAIRED_DESKTOP | // handle desktop-entry files without metainfo data
ComposeFlags.NO_FINAL_CHECK // we trigger the final check manually
);
// we handle all threading, so the compose process doesn't also have to be threaded
compose.removeFlags (ComposeFlags.USE_THREADS);
if (l10nUnit !is null)
compose.setLocaleUnit (l10nUnit);
// set max screenshot size in bytes, if size is limited
if (conf.maxScrFileSize != 0)
compose.setMaxScreenshotSize (conf.maxScrFileSize * 1024 * 1024);
// enable or disable user-defined features
if (conf.feature.validate)
compose.addFlags (ComposeFlags.VALIDATE);
else
compose.removeFlags (ComposeFlags.VALIDATE);
if (conf.feature.noDownloads)
compose.removeFlags (ComposeFlags.ALLOW_NET);
else
compose.addFlags (ComposeFlags.ALLOW_NET);
if (conf.feature.processLocale)
compose.addFlags (ComposeFlags.PROCESS_TRANSLATIONS);
else
compose.removeFlags (ComposeFlags.PROCESS_TRANSLATIONS);
if (conf.feature.processFonts)
compose.addFlags (ComposeFlags.PROCESS_FONTS);
else
compose.removeFlags (ComposeFlags.PROCESS_FONTS);
if (conf.feature.screenshotVideos)
compose.addFlags (ComposeFlags.ALLOW_SCREENCASTS);
else
compose.removeFlags (ComposeFlags.ALLOW_SCREENCASTS);
// guess an icon policy that matches the user settings best
bool hasIconsRemote = false;
bool hasIconsCached = false;
foreach (const ref policy; conf.iconSettings) {
if (policy.storeCached)
hasIconsCached = true;
if (policy.storeRemote)
hasIconsRemote = true;
if (hasIconsRemote && hasIconsCached)
break;
}
compose.setIconPolicy (IconPolicy.BALANCED);
if (!hasIconsRemote && hasIconsCached)
compose.setIconPolicy (IconPolicy.ONLY_CACHED);
if (!hasIconsCached && hasIconsRemote)
compose.setIconPolicy (IconPolicy.ONLY_REMOTE);
// register allowed custom keys with the composer
foreach (const ref key; conf.allowedCustomKeys.byKey)
compose.addCustomAllowed (key);
}
/**
* Helper function for DataExtractor.parseDesktopFile
*/
extern(C)
static void checkMetadataIntermediate (AscResult *cres, AscUnit *cunit, void *userData)
{
import ascompose.Result : Result;
import asgen.config : EXTRA_METAINFO_FAKE_PKGNAME;
auto self = cast(DataExtractor) userData;
auto result = new Result (cres);
auto cptsPtrArray = result.fetchComponents ();
for (uint i = 0; i < cptsPtrArray.len; i++) {
auto cpt = new Component (cast (AsComponent*) cptsPtrArray.index (i));
auto gcid = result.gcidForComponent (cpt);
// don't run expensive operations if the metadata already exists
auto existingMData = self.dstore.getMetadata (self.dtype, gcid);
if (existingMData is null)
continue;
// To account for packages which change their package name, we
// also need to check if the package this component is associated
// with matches ours.
// If it doesn't, we can't just link the package to the component.
bool samePkg = false;
immutable bundleId = result.getBundleId;
if (bundleId == EXTRA_METAINFO_FAKE_PKGNAME) {
// the fake package is exempt from this check
samePkg = true;
} else {
if (self.dtype == DataType.YAML) {
if (existingMData.canFind (format ("Package: %s\n", bundleId)))
samePkg = true;
} else {
if (existingMData.canFind (format ("%s", bundleId)))
samePkg = true;
}
}
if ((!samePkg) && (cpt.getKind != ComponentKind.WEB_APP)) {
// The exact same metadata exists in a different package already, we emit an error hint.
// ATTENTION: This does not cover the case where *different* metadata (as in, different summary etc.)
// but with the *same ID* exists.
// We only catch that kind of problem later.
auto cdata = new Metadata ();
cdata.setFormatStyle (FormatStyle.COLLECTION);
cdata.setFormatVersion (self.conf.formatVersion);
if (self.dtype == DataType.YAML)
cdata.parse (existingMData, FormatKind.YAML);
else
cdata.parse (existingMData, FormatKind.XML);
auto ecpt = cdata.getComponent ();
const pkgNames = ecpt.getPkgnames;
string pkgName = "(none)";
if (!pkgNames.empty)
pkgName = pkgNames[0];
result.addHint (cpt, "metainfo-duplicate-id", ["cid", cpt.getId,
"pkgname", pkgName]);
}
// drop the component as we already have processed it, but keep its
// global ID so we can still register the ID with this package.
result.removeComponentFull(cpt, false);
}
}
/**
* Helper function for DataExtractor.processPackage
*/
extern(C)
static GPtrArray *translateDesktopTextCallback (GKeyFile *dePtr, const(char) *text, void *userData)
{
import glib.KeyFile : KeyFile;
import glib.c.functions;
import std.string : fromStringz, toStringz;
auto pkg = *cast(Package*) userData;
auto de = new KeyFile (dePtr, false);
auto res = g_ptr_array_new_with_free_func (&g_free);
auto translations = pkg.getDesktopFileTranslations (de, cast(string) text.fromStringz);
foreach (ref key, ref value; translations) {
g_ptr_array_add (res, g_strdup (key.toStringz));
g_ptr_array_add (res, g_strdup (value.toStringz));
}
return res;
}
GeneratorResult processPackage (Package pkg)
{
import ascompose.Result : Result;
// reset compose instance to clear data from any previous invocation
compose.reset ();
// set external desktop-entry translation function, if needed
immutable externalL10n = pkg.hasDesktopFileTranslations;
compose.setDesktopEntryL10nFunc (externalL10n? &translateDesktopTextCallback : null,
externalL10n? &pkg : null);
// wrap package into unit, so AppStream Compose can work with it
auto unit = new PackageUnit (pkg);
compose.addUnit (unit);
// process all data
compose.run (null);
auto resultsArray = compose.getResults ();
// we processed one unit, so should always generate one result
if (resultsArray.len != 1) {
logError ("Expected %s result for data extraction, but retrieved %s.", 1, resultsArray.len);
assert (resultsArray.len == 1);
}
// create result wrapper
auto gres = GeneratorResult (new Result (cast (AscResult*) resultsArray.index (0)),
pkg);
// process icons and perform additional refinements
auto cptsPtrArray = gres.fetchComponents ();
for (uint i = 0; i < cptsPtrArray.len; i++) {
auto cpt = new Component (cast (AsComponent*) cptsPtrArray.index (i));
immutable ckind = cpt.getKind;
// find & store icons
iconh.process (gres, cpt);
if (gres.isIgnored (cpt))
continue;
// add fallback long descriptions only for desktop apps, console apps and web apps
if (cpt.getMergeKind != MergeKind.NONE)
continue;
if (ckind != ComponentKind.DESKTOP_APP && ckind != ComponentKind.CONSOLE_APP && ckind != ComponentKind.WEB_APP)
continue;
// inject package descriptions, if needed
auto flags = cpt.getValueFlags;
cpt.setValueFlags (flags | AsValueFlags.NO_TRANSLATION_FALLBACK);
cpt.setActiveLocale ("C");
if (!cpt.getDescription.empty)
continue;
// component doesn't have a long description, add one from the packaging.
auto desc_added = false;
foreach (ref lang, ref desc; pkg.description) {
cpt.setDescription (desc, lang);
desc_added = true;
}
if (desc_added) {
// we only add the "description-from-package" tag if we haven't alreaey
// emitted a "no-metainfo" tag, to avoid two hints explaining the same thing
if (!gres.hasHint (cpt, "no-metainfo")) {
if (!gres.addHint (cpt, "description-from-package"))
continue;
}
} else {
if (!gres.addHint (cpt, "description-missing", ["kind": AsUtils.componentKindToString (ckind)]))
continue;
}
}
// handle GStreamer integration (usually for Ubuntu)
if (conf.feature.processGStreamer && !pkg.gst.isNull && pkg.gst.get.isNotEmpty) {
auto data = appender!string;
data.reserve(200);
auto cpt = new Component ();
cpt.setId (pkg.name);
cpt.setKind (ComponentKind.CODEC);
cpt.setName ("GStreamer Multimedia Codecs", "C");
foreach (ref lang, ref desc; pkg.summary) {
cpt.setSummary (desc, lang);
data ~= desc;
}
gres.addComponent (cpt, data.data.toStaticGBytes);
}
// perform final checks
compose.finalizeResults ();
// do our own final validation
cptsPtrArray = gres.fetchComponents ();
for (uint i = 0; i < cptsPtrArray.len; i++) {
auto cpt = new Component (cast (AsComponent*) cptsPtrArray.index (i));
immutable ckind = cpt.getKind;
if (cpt.getMergeKind != MergeKind.NONE)
continue;
if (cpt.getPkgnames.empty) {
// no packages are associated with this component
if (ckind != ComponentKind.WEB_APP &&
ckind != ComponentKind.OPERATING_SYSTEM &&
ckind != ComponentKind.REPOSITORY) {
// this component is not allowed to have no installation candidate
if (!cpt.hasBundle) {
if (!gres.addHint (cpt, "no-install-candidate"))
continue;
}
}
} else {
// packages are associated with this component
if (pkg.kind == PackageKind.FAKE) {
import std.array : array;
import asgen.config : EXTRA_METAINFO_FAKE_PKGNAME;
import std.algorithm.iteration : filter;
// drop any association with the dummy package
auto pkgnames = cpt.getPkgnames;
cpt.setPkgnames (array(pkgnames.filter!(a => a != EXTRA_METAINFO_FAKE_PKGNAME)));
}
}
}
// clean up and return result
pkg.finish ();
return gres;
}
}
appstream-generator-0.8.6/src/asgen/hintregistry.d 0000664 0000000 0000000 00000013454 14160667064 0022312 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2021 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.hint;
@safe:
import std.stdio;
import std.string;
import std.json;
import std.conv : to;
import appstream.Validator : Validator;
import appstream.c.types : IssueSeverity;
import ascompose.Hint : Hint;
import ascompose.Globals : Globals;
static import appstream.Utils;
alias AsUtils = appstream.Utils.Utils;
import asgen.logging;
import asgen.utils;
/**
* Each issue hint type has a severity assigned to it:
* ERROR: A fatal error which resulted in the component being excluded from the final metadata.
* WARNING: An issue which did not prevent generating meaningful data, but which is still serious
* and should be fixed (warning of this kind usually result in less data).
* INFO: Information, no immediate action needed (but will likely be an issue later).
* PEDANTIC: Information which may improve the data, but could also be ignored.
*/
/**
* Definition of a issue hint.
*/
struct HintDefinition
{
string tag; /// Unique issue tag
IssueSeverity severity; /// Issue severity
string explanation; /// Explanation template
}
/**
* Load all issue hints from file and register them globally.
*/
void loadHintsRegistry () @trusted
{
import std.path;
static import std.file;
// find the hint definition file
auto hintsDefFile = getDataPath ("asgen-hints.json");
if (!std.file.exists (hintsDefFile)) {
logError ("Hints definition file '%s' was not found! This means we can not determine severity of issue tags and not render report pages.", hintsDefFile);
return;
}
// read the hints definition JSON file
auto f = File (hintsDefFile, "r");
string jsonData;
string line;
while ((line = f.readln ()) !is null)
jsonData ~= line;
auto hintDefsJSON = parseJSON (jsonData);
bool checkAlreadyLoaded = true;
foreach (ref tag; hintDefsJSON.object.byKey) {
auto j = hintDefsJSON[tag];
immutable severity = AsUtils.severityFromString (j["severity"].str);
bool overrideExisting = false;
if ((tag == "icon-not-found") ||
(tag == "internal-unknown-tag") ||
(tag == "internal-error") ||
(tag == "no-metainfo"))
overrideExisting = true;
if (checkAlreadyLoaded) {
if (!overrideExisting && Globals.hintTagSeverity (tag) != IssueSeverity.UNKNOWN) {
logDebug ("Global hints registry already loaded.");
break;
}
checkAlreadyLoaded = false;
}
string explanation = "";
if (j["text"].type == JSONType.array) {
foreach (l; j["text"].array)
explanation ~= l.str ~ "\n";
} else {
explanation = j["text"].str;
}
if (!Globals.addHintTag (tag, severity, explanation, overrideExisting))
logError ("Unable to override existing hint tag %s.", tag);
}
}
/**
* Save information about all hint templates we know about to a JSON file.
*/
void saveHintsRegistryToJsonFile (const string fname) @trusted
{
// FIXME: is this really the only way you can set a type for JSONValue?
auto map = JSONValue (["null": 0]);
map.object.remove ("null");
foreach (const htag; Globals.getHintTags) {
const hdef = retrieveHintDef (htag);
auto jval = JSONValue (["text": JSONValue (hdef.explanation),
"severity": JSONValue (AsUtils.severityToString (hdef.severity))]);
map.object[hdef.tag] = jval;
}
File file = File(fname, "w");
file.writeln (map.toJSON (true));
file.close ();
}
HintDefinition retrieveHintDef (string tag) @trusted
{
HintDefinition hdef;
hdef.tag = tag;
hdef.severity = Globals.hintTagSeverity (tag);
if (hdef.severity == IssueSeverity.UNKNOWN)
return HintDefinition ();
hdef.explanation = Globals.hintTagExplanation (tag);
return hdef;
}
auto toJsonValue (Hint hint) @trusted
{
auto hintList = hint.getExplanationVarsList;
string[string] vars;
for (uint i = 0; i < hintList.len; i++) {
if (i % 2 != 0)
continue;
const auto key = fromStringz (cast(char*) hintList.index (i)).to!string;
const auto value = fromStringz (cast(char*) hintList.index (i + 1)).to!string;
vars[key] = value;
}
return JSONValue(["tag": JSONValue (hint.getTag),
"vars": JSONValue (vars)]);
}
@trusted
unittest
{
import std.exception : assertThrown;
import glib.GException : GException;
writeln ("TEST: ", "Issue Hints");
assertThrown!GException (new Hint ("description-from-package"));
loadHintsRegistry ();
auto hint = new Hint ("description-from-package");
foreach (k, v; ["rainbows": "yes", "unicorns": "no", "storage": "towel"])
hint.addExplanationVar (k, v);
auto root = hint.toJsonValue ();
writeln (root.toJSON (true));
assert (retrieveHintDef ("asv-relation-item-invalid-vercmp").severity != IssueSeverity.UNKNOWN);
saveHintsRegistryToJsonFile ("/tmp/testsuite-asgen-hints.json");
}
appstream-generator-0.8.6/src/asgen/iconhandler.d 0000664 0000000 0000000 00000110711 14160667064 0022037 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2021 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.iconhandler;
import std.stdio : File, writeln;
import std.string : endsWith, startsWith, format;
import std.array : replace, array, empty;
import std.path : baseName, buildPath;
import std.uni : toLower;
import std.file : mkdirRecurse;
import std.algorithm : canFind, map;
import std.variant : Algebraic;
import std.typecons : scoped;
import std.parallelism : parallel;
import std.concurrency : Generator, yield;
import glib.KeyFile : KeyFile;
import glib.GException : GException;
import appstream.Component;
import appstream.Icon;
static import ascompose.Utils;
alias AscUtils = ascompose.Utils.Utils;
import ascompose.Image : Image;
import ascompose.Canvas : Canvas;
import ascompose.c.types : ImageFormat, ImageLoadFlags, ImageSaveFlags;
static import std.file;
import asgen.utils;
import asgen.logging;
import asgen.result;
import asgen.backends.interfaces;
import asgen.contentsstore;
import asgen.config : Config, IconPolicy;
// all image extensions that we recognize as possible for icons.
// the most favorable file extension needs to come first to prefer it
private immutable possibleIconExts = [".png", ".jpg", ".svgz", ".svg", ".gif", ".ico", ".xpm"];
// the image extensions that we will actually allow software to have.
private immutable allowedIconExts = [".png", ".jpg", ".svgz", ".svg", ".xpm"];
/**
* Describes an icon theme as specified in the XDG theme spec.
*/
private final class Theme
{
private:
string _name;
Algebraic!(int, string)[string][] _directories;
public:
this (const string name, const(ubyte)[] indexData)
{
this._name = name;
auto index = new KeyFile ();
auto indexText = cast(string) indexData;
index.loadFromData (indexText, -1, GKeyFileFlags.NONE);
size_t dummy;
foreach (section; index.getGroups (dummy)) {
string type;
string context;
int scale;
int threshold;
int size;
int minSize;
int maxSize;
// we ignore symbolic icons
if (section.startsWith("symbolic/"))
continue;
try {
size = index.getInteger (section, "Size");
context = index.getString (section, "Context");
} catch (GException) {
continue;
}
try {
threshold = index.getInteger (section, "Threshold");
} catch (GException) {
threshold = 2;
}
try {
type = index.getString (section, "Type");
} catch (GException) {
type = "Threshold";
}
try {
minSize = index.getInteger (section, "MinSize");
} catch (GException) {
minSize = size;
}
try {
maxSize = index.getInteger (section, "MaxSize");
} catch (GException) {
maxSize = size;
}
try {
scale = index.getInteger (section, "Scale");
} catch (GException) {
scale = 1;
}
if (size == 0)
continue;
auto themedir = [
"path": Algebraic!(int, string) (section),
"type": Algebraic!(int, string) (type),
"size": Algebraic!(int, string) (size),
"minsize": Algebraic!(int, string) (minSize),
"maxsize": Algebraic!(int, string) (maxSize),
"threshold": Algebraic!(int, string) (threshold),
"scale": Algebraic!(int, string) (scale)
];
_directories ~= themedir;
}
// sort our directory list, so the smallest size is at the top
import std.algorithm : sort;
_directories.sort!("a[\"size\"].get!int < b[\"size\"].get!int");
}
this (const string name, Package pkg)
{
auto indexData = pkg.getFileData (buildPath ("/usr/share/icons", name, "index.theme"));
this (name, indexData);
}
@property
string name () const
{
return this._name;
}
@property
auto directories () const
{
return this._directories;
}
/**
* Check if a directory is suitable for the selected size.
* If @assumeThresholdScalable is set to true, we will allow
* downscaling of any higher-than-requested icon size, even if the
* section is of "Threshold" type and would usually prohibit the scaling.
*/
package bool directoryMatchesSize (const Algebraic!(int, string)[string] themedir, ImageSize size, bool assumeThresholdScalable = false)
{
immutable scale = themedir["scale"].get!int;
if (scale != size.scale)
return false;
immutable type = themedir["type"].get!string;
if (type == "Fixed")
return size.toInt () == themedir["size"].get!int;
if (type == "Scalable") {
if ((themedir["minsize"].get!int <= size.toInt) && (size.toInt <= themedir["maxsize"].get!int))
return true;
return false;
}
if (type == "Threshold") {
immutable themeSize = themedir["size"].get!int;
immutable th = themedir["threshold"].get!int;
if (assumeThresholdScalable) {
// we treat this "Threshold" as if we were allowed to downscale its icons if they
// have a higher size.
// This can lead to "wrong" scaling, but allows us to retrieve more icons.
return themeSize >= size.toInt;
} else {
// follow the proper algorithm as defined by the XDG spec
if (((themeSize - th) <= size.toInt) && (size.toInt <= (themeSize + th)))
return true;
}
return false;
}
return false;
}
/**
* Returns an iteratable of possible icon filenames that match @iname and @size.
* If @relaxedScalingRules is set to true, we scale down any bigger icon seize, even
* if the theme definition would usually prohibit that.
**/
auto matchingIconFilenames (string iname, ImageSize size, bool relaxedScalingRules = false)
{
struct IconFilenamesRange
{
// state
int idxThemeDir = 0;
int idxExt = 0;
bool isEmpty = false;
// extensions to check for
immutable extensions = ["png", "svgz", "svg", "xpm"];
// reference to the theme
Theme theme;
this (Theme theme)
{
this.theme = theme;
// get initial matching icon, as
// front() is called before popFront()
idxExt = -1;
popFront ();
}
bool empty () @property
{
return idxThemeDir >= theme.directories.length;
}
string front () const @property
{
return "/usr/share/icons/%s/%s/%s.%s".format (theme.name,
theme.directories[idxThemeDir]["path"].get!(string),
iname,
extensions[idxExt]);
}
void popFront ()
{
do {
idxExt++;
if (idxExt >= extensions.length) {
idxExt = 0;
idxThemeDir++;
}
if (idxThemeDir >= theme.directories.length)
return; // end of range reached
if (theme.directoryMatchesSize (theme.directories[idxThemeDir], size, relaxedScalingRules))
return;
} while (true);
}
}
return IconFilenamesRange(this);
}
}
/**
* Finds icons in a software archive and stores them in the
* correct sizes for a given AppStream component.
*/
final class IconHandler
{
private:
string mediaExportPath;
Theme[] themes;
Package[string] iconFiles;
string[] themeNames;
IconPolicy[] iconPolicy;
IconPolicy defaultIconPolicy;
bool allowIconUpscaling;
bool allowRemoteIcons;
public:
this (ContentsStore ccache, string mediaPath, IconPolicy[] iconPolicy, Package[string] pkgMap, string iconTheme = null)
{
logDebug ("Creating new IconHandler");
iconFiles.clear ();
mediaExportPath = mediaPath;
this.iconPolicy = iconPolicy;
foreach (ref policy; iconPolicy) {
if (policy.iconSize == ImageSize (64)) {
defaultIconPolicy = policy;
break;
}
}
// Sanity checks
if (defaultIconPolicy.iconSize != ImageSize (64))
throw new Exception ("Could not find default icon site '64x64' in icon policy list. This is a bug in the generator or configuration file.");
assert (defaultIconPolicy.storeCached == true);
auto conf = Config.get;
allowIconUpscaling = conf.feature.allowIconUpscale;
// we assume that when screenshot storage is permitted, remote icons are also okay
allowRemoteIcons = conf.feature.storeScreenshots && !conf.mediaBaseUrl.empty;
// Preseeded theme names.
// * prioritize hicolor, because that's where apps often install their upstream icon
// * then look at the theme given in the config file
// * allow Breeze icon theme, needed to support KDE apps (they have no icon at all, otherwise...)
// * in rare events, GNOME needs the same treatment, so special-case Adwaita as well
// * We need at least one icon theme to provide the default XDG icon spec stock icons.
// A fair take would be to select them between KDE and GNOME at random, but for consistency and
// because everyone hates unpredictable behavior, we sort alphabetically and prefer Adwaita over Breeze.
themeNames = ["hicolor"];
if (iconTheme !is null)
themeNames ~= iconTheme;
themeNames ~= "Adwaita"; // GNOME
themeNames ~= "breeze"; // KDE
Package getPackage (string pkid)
{
if (pkid is null)
return null;
return pkgMap.get (pkid, null);
}
// load data from the contents index.
// we don't show mercy to memory here, we just want the icon lookup to be fast,
// so we have to cache the data.
Theme[string] tmpThemes;
auto filesPkids = ccache.getIconFilesMap (array(pkgMap.byKey));
foreach (info; parallel (filesPkids.byKeyValue, 100)) {
immutable fname = info.key;
immutable pkgid = info.value;
if (fname.startsWith ("/usr/share/pixmaps/")) {
auto pkg = getPackage (pkgid);
if (pkg is null)
continue;
synchronized (this) iconFiles[fname] = pkg;
continue;
}
// optimization: check if we actually have an interesting path before
// entering the foreach loop.
if (!fname.startsWith ("/usr/share/icons/"))
continue;
auto pkg = getPackage (filesPkids[fname]);
if (pkg is null)
continue;
foreach (name; themeNames) {
if (fname == "/usr/share/icons/%s/index.theme".format (name)) {
synchronized (this) tmpThemes[name] = new Theme (name, pkg);
} else if (fname.startsWith ("/usr/share/icons/%s".format (name))) {
synchronized (this) iconFiles[fname] = pkg;
}
}
}
// when running on partial repos (e.g. PPAs) we might not have a package containing the
// hicolor theme definition. Since we always need it to be there to properly process icons,
// we inject our own copy here.
if ("hicolor" !in tmpThemes) {
logInfo ("No packaged hicolor icon theme found, using built-in one.");
auto hicolorThemeIndex = getDataPath ("hicolor-theme-index.theme");
if (!std.file.exists (hicolorThemeIndex)) {
logError ("Hicolor icon theme index at '%s' was not found! We will not be able to handle icons in this theme.", hicolorThemeIndex);
} else {
ubyte[] indexData;
auto f = File (hicolorThemeIndex, "r");
while (!f.eof) {
char[GENERIC_BUFFER_SIZE] buf;
indexData ~= f.rawRead (buf);
}
tmpThemes["hicolor"] = new Theme ("hicolor", indexData);
}
}
// this is necessary to keep the ordering (and therefore priority) of themes.
// we don't know the order in which we find index.theme files in the code above,
// therefore this sorting is necessary.
foreach (tname; themeNames) {
if (tname in tmpThemes)
themes ~= tmpThemes[tname];
}
logDebug ("Created new IconHandler.");
}
private string getIconNameAndClear (Component cpt)
{
string name = null;
// a not-processed icon name is stored as "1x1px" icon, so we can
// quickly identify it here.
auto icon = componentGetRawIcon (cpt);
if (!icon.isNull) {
if (icon.get.getKind == IconKind.LOCAL)
name = icon.get.getFilename ();
else
name = icon.get.getName ();
}
// clear the list of icons in this component
auto iconsArray = cpt.getIcons ();
if (iconsArray.len > 0)
iconsArray.removeRange (0, iconsArray.len);
return name;
}
static private bool iconAllowed (string iconName)
{
foreach (ref ext; allowedIconExts)
if (iconName.endsWith (ext))
return true;
return false;
}
/**
* Generates potential filenames of the icon that is searched for in the
* given size.
**/
private auto possibleIconFilenames (string iconName, ImageSize size, bool relaxedScalingRules = false)
{
auto gen = new Generator!string (
{
foreach (theme; this.themes) {
foreach (fname; theme.matchingIconFilenames (iconName, size, relaxedScalingRules))
yield (fname);
}
if (size.scale == 1 && size.width == 64) {
// Check icons root directory for icon files
// this is "wrong", but we support it for compatibility reasons.
// However, we only ever use it to satisfy the 64x64px requirement
foreach (extension; possibleIconExts)
yield ("/usr/share/icons/%s%s".format (iconName, extension));
// check pixmaps directory for icons
// we only ever use the pixmap directory contents to satisfy the minimum 64x64px icon
// requirement. Otherwise we get weird upscaling to higher sizes or HiDPI sizes happening,
// as later code tries to downscale "bigger" sizes.
foreach (extension; possibleIconExts)
yield ("/usr/share/pixmaps/%s%s".format (iconName, extension));
}
});
return gen;
}
/**
* Helper structure for the findIcons
* method.
**/
private struct IconFindResult
{
Package pkg;
string fname;
this (Package pkg, string fname) {
this.pkg = pkg;
this.fname = fname;
}
}
/**
* Looks up 'icon' with 'size' in popular icon themes according to the XDG
* icon theme spec.
**/
private auto findIcons (string iconName, const ImageSize[] sizes, Package pkg = null)
{
IconFindResult[ImageSize] sizeMap;
foreach (size; sizes) {
// search for possible icon filenames, using relaxed scaling rules by default
foreach (fname; possibleIconFilenames (iconName, size, true)) {
if (pkg !is null) {
// we are supposed to search in one particular package
if (pkg.contents.canFind (fname)) {
sizeMap[size] = IconFindResult (pkg, fname);
break;
}
} else {
// global search in all packages
pkg = iconFiles.get (fname, null);
// continue if filename is not in map
if (pkg is null)
continue;
sizeMap[size] = IconFindResult (pkg, fname);
break;
}
}
}
return sizeMap;
}
/**
* Strip file extension from icon.
*/
string stripIconExt (ref string iconName)
{
if (iconName.endsWith (".png"))
return iconName[0..$-4];
if (iconName.endsWith (".svg"))
return iconName[0..$-4];
if (iconName.endsWith (".xpm"))
return iconName[0..$-4];
if (iconName.endsWith (".svgz"))
return iconName[0..$-5];
return iconName;
}
/**
* Extracts the icon from the package and stores it in the cache.
* Ensures the stored icon always has the size given in "size", and renders
* scalable vectorgraphics if necessary.
*
* Params:
* cpt = The component this icon belongs to.
* res = The result the component belongs to.
* cptExportPath = The data export directory of the component.
* sourcePkg = The package the to-be-extracted icon is located in.
* iconPath = The (absolute) path to the icon.
* size = The size the icon should be stored in.
**/
private bool storeIcon (Component cpt,
ref GeneratorResult gres,
string cptExportPath,
Package sourcePkg,
string iconPath,
IconPolicy policy)
{
immutable size = policy.iconSize;
auto iformat = AscUtils.imageFormatFromFilename (iconPath);
if (iformat == ImageFormat.UNKNOWN) {
gres.addHint (cpt.getId (), "icon-format-unsupported", ["icon_fname": baseName (iconPath)]);
return false;
}
auto path = buildPath (cptExportPath, "icons", size.toString);
auto iconName = (gres.pkg.kind == PackageKind.FAKE)? baseName (iconPath) : "%s_%s".format (gres.pkg.name, baseName (iconPath));
if (iconName.endsWith (".svgz"))
iconName = iconName.replace (".svgz", ".png");
else if (iconName.endsWith (".svg"))
iconName = iconName.replace (".svg", ".png");
else if (iconName.endsWith (".xpm"))
iconName = iconName.replace (".xpm", ".png");
auto iconStoreLocation = buildPath (path, iconName);
if (std.file.exists (iconStoreLocation)) {
// we already extracted that icon, skip the extraction step
// and just add the new icon.
if (policy.storeCached) {
auto icon = new Icon ();
icon.setKind (IconKind.CACHED);
icon.setWidth (size.width);
icon.setHeight (size.height);
icon.setScale (size.scale);
icon.setName (iconName);
cpt.addIcon (icon);
}
if (policy.storeRemote && allowRemoteIcons) {
immutable gcid = gres.gcidForComponent (cpt);
if (gcid is null) {
gres.addHint (cpt, "internal-error", "No global ID could be found for the component, could not add remote icon.");
return true;
}
immutable remoteIconUrl = buildPath (gcid, "icons", size.toString, iconName);
auto icon = new Icon ();
icon.setKind (IconKind.REMOTE);
icon.setWidth (size.width);
icon.setHeight (size.height);
icon.setScale (size.scale);
icon.setUrl (remoteIconUrl);
cpt.addIcon (icon);
}
return true;
}
// filepath is checked because icon can reside in another binary
// eg amarok's icon is in amarok-data
const(ubyte)[] iconData = null;
try {
iconData = sourcePkg.getFileData (iconPath);
} catch (Exception e) {
gres.addHint(cpt.getId (), "pkg-extract-error", ["fname": baseName (iconPath),
"pkg_fname": baseName (sourcePkg.getFilename),
"error": e.msg]);
return false;
}
if (iconData.empty ()) {
gres.addHint (cpt.getId (), "pkg-empty-file", ["fname": baseName (iconPath),
"pkg_fname": baseName (sourcePkg.getFilename)]);
return false;
}
auto scaled_width = size.width * size.scale;
auto scaled_height = size.height * size.scale;
if ((iformat == ImageFormat.SVG) || (iformat == ImageFormat.SVGZ)) {
// create target directory
mkdirRecurse (path);
try {
import gio.MemoryInputStream : MemoryInputStream;
auto stream = new MemoryInputStream (cast(ubyte[]) iconData, null);
auto cv = scoped!Canvas (scaled_width, scaled_height);
cv.renderSvg (stream);
cv.savePng (iconStoreLocation);
} catch (Exception e) {
gres.addHint(cpt.getId (), "image-write-error", ["fname": baseName (iconPath),
"pkg_fname": baseName (sourcePkg.getFilename),
"error": e.msg]);
return false;
}
} else {
Image img;
try {
img = new Image ((cast(ubyte[]) iconData).ptr, cast(ptrdiff_t)iconData.length,
0, iconPath.endsWith (".svgz"), ImageLoadFlags.NONE);
} catch (Exception e) {
gres.addHint(cpt.getId (), "image-write-error", ["fname": baseName (iconPath),
"pkg_fname": baseName (sourcePkg.getFilename),
"error": e.msg]);
return false;
}
if (iformat == ImageFormat.XPM) {
// we use XPM images only if they are large enough
if (allowIconUpscaling) {
// we only try upscaling for the default 64x64px size and only if
// the icon is not too small
if (size != ImageSize (64))
return false;
if ((img.getWidth < 48) || (img.getHeight < 48))
return false;
} else {
if ((img.getWidth < scaled_width) || (img.getHeight < scaled_height))
return false;
}
}
// ensure that we don't try to make an application visible that has a really tiny icon
// by upscaling it to a blurry mess
if (size.scale == 1 && size.width == 64) {
if ((img.getWidth < 48) || (img.getHeight < 48)) {
gres.addHint (cpt, "icon-too-small", ["icon_name": iconName,
"icon_size": "%ux%u".format (img.getWidth, img.getHeight)]);
return false;
}
}
// warn about icon upscaling, it looks ugly
if (scaled_width > img.getWidth) {
gres.addHint (cpt, "icon-scaled-up", ["icon_name": iconName,
"icon_size": "%ux%u".format (img.getWidth, img.getHeight),
"scale_size": size.toString]);
}
// create target directory
mkdirRecurse (path);
try {
img.scale (scaled_width, scaled_height);
img.saveFilename (iconStoreLocation,
0, 0,
ImageSaveFlags.OPTIMIZE);
} catch (Exception e) {
gres.addHint (cpt, "image-write-error", ["fname": baseName (iconPath),
"pkg_fname": baseName (sourcePkg.getFilename),
"error": e.msg]);
return false;
}
}
if (policy.storeCached) {
auto icon = new Icon ();
icon.setKind (IconKind.CACHED);
icon.setWidth (size.width);
icon.setHeight (size.height);
icon.setScale (size.scale);
icon.setName (iconName);
cpt.addIcon (icon);
}
if (policy.storeRemote && allowRemoteIcons) {
immutable gcid = gres.gcidForComponent (cpt);
if (gcid is null) {
gres.addHint (cpt, "internal-error", "No global ID could be found for the component, could not add remote icon.");
return true;
}
immutable remoteIconUrl = buildPath (gcid, "icons", size.toString, iconName);
auto icon = new Icon ();
icon.setKind (IconKind.REMOTE);
icon.setWidth (size.width);
icon.setHeight (size.height);
icon.setScale (size.scale);
icon.setUrl (remoteIconUrl);
cpt.addIcon (icon);
}
return true;
}
/**
* Helper function to try to find an icon that we can up- or downscale to the desired size.
*/
private auto findIconScalableToSize (ref IconFindResult[ImageSize] possibleIcons, const ref ImageSize size)
{
IconFindResult info;
info.pkg = null;
// on principle, never attempt to up- or downscale an icon to something below
// AppStream's default icon size.
// The clients can do that just as well, without us wasting disk space
// and network bandwidth.
if (size.scale == 1 && size.width < 64)
return info;
// the size we want wasn't found, can we downscale a larger one?
foreach (ref asize; possibleIcons.byKey) {
auto data = possibleIcons[asize];
if (asize.scale != size.scale)
continue;
if (asize < size)
continue;
info = data;
break;
}
if ((info.pkg is null) && (allowIconUpscaling) && (size == defaultIconPolicy.iconSize)) {
// no icon was found to downscale, but we allow upscaling, so try one last time
// to find a suitable icon for at least the default AppStream icon size.
foreach (ref asize; possibleIcons.byKey) {
auto data = possibleIcons[asize];
// we never allow icons smaller than 48x48px
if (asize.width < 48)
continue;
if (asize.scale != size.scale)
continue;
info = data;
break;
}
}
return info;
}
/**
* Try to find & store icons for a selected component.
*/
bool process (ref GeneratorResult gres, Component cpt)
{
// we don't touch fonts unless those didn't have their icon
// rendered from the font itself already
if (cpt.getKind == ComponentKind.FONT) {
auto iconsArr = cpt.getIcons ();
for (uint i = 0; i < iconsArr.len; i++) {
auto icon = new Icon (cast(AsIcon*) iconsArr.index (i));
// nothing to do for us if cached and remote icons are already there
if (icon.getKind == IconKind.CACHED || icon.getKind == IconKind.REMOTE)
return true;
}
}
auto iconName = getIconNameAndClear (cpt);
// nothing to do if there is no icon
if (iconName is null)
return true;
auto gcid = gres.gcidForComponent (cpt);
if (gcid is null) {
auto cid = cpt.getId ();
if (cid is null)
cid = "general";
gres.addHint (cid, "internal-error", "No global ID could be found for the component.");
return false;
}
auto cptMediaPath = buildPath (mediaExportPath, gcid);
if (iconName.startsWith ("/")) {
logDebug ("Looking for icon '%s' for '%s::%s' (path)", iconName, gres.pkid, cpt.getId);
if (gres.pkg.contents.canFind (iconName))
return storeIcon (cpt, gres, cptMediaPath, gres.pkg, iconName, defaultIconPolicy);
// we couldn't find the absolute icon path
gres.addHint (cpt.getId, "icon-not-found", ["icon_fname": iconName]);
return false;
} else {
logDebug ("Looking for icon '%s' for '%s::%s' (XDG)", iconName, gres.pkid, cpt.getId);
iconName = baseName (iconName);
// Small hack: Strip .png and other extensions from icon files to make the XDG and Pixmap finder
// work properly, which add their own icon extensions and find the most suitable icon.
iconName = stripIconExt (iconName);
string lastIconName = null;
/// Search for an icon in XDG icon directories.
/// Returns true on success and sets lastIconName to the
/// last icon name that has been handled.
bool findAndStoreXdgIcon (Package epkg = null)
{
auto iconRes = findIcons (iconName, array(iconPolicy.map!(a => a.iconSize)), epkg);
if (iconRes.empty)
return false;
IconFindResult[ImageSize] iconsStored;
foreach (ref policy; iconPolicy) {
immutable size = policy.iconSize;
auto infoP = (size in iconRes);
IconFindResult info;
if (infoP is null)
info.pkg = null;
else
info = *infoP;
// check if we can scale another size to the desired one
if (info.pkg is null)
info = findIconScalableToSize (iconRes, size);
// give up if we still haven't found an icon (in which case `info.pkg` would be set)
if (info.pkg is null)
continue;
lastIconName = info.fname;
if (iconAllowed (lastIconName)) {
if (storeIcon (cpt, gres, cptMediaPath, info.pkg, lastIconName, policy))
iconsStored[size] = info;
} else {
// the found icon is not suitable, but maybe we can scale a differently sized icon to the right one?
info = findIconScalableToSize (iconRes, size);
if (info.pkg is null)
continue;
if (iconAllowed (info.fname)) {
if (storeIcon (cpt, gres, cptMediaPath, info.pkg, lastIconName, policy))
iconsStored[size] = info;
lastIconName = info.fname;
}
}
if (gres.isIgnored (cpt)) {
// running storeIcon() in this loop may lead to rejection
// of this component, in case icons can't be saved.
// we just give up in that case.
return false;
}
}
// ensure we have stored a 64x64px icon, since this is mandated
// by the AppStream spec by downscaling a larger icon that we
// might have found.
if (ImageSize(64) in iconsStored) {
logDebug ("Found icon %s - %s in XDG directories, 64x64px size is present", gres.pkid, iconName);
return true;
} else {
foreach (size; iconPolicy.map!(a => a.iconSize)) {
if (size !in iconsStored)
continue;
if (size < ImageSize(64))
continue;
logInfo ("Downscaling icon %s - %s from %s to %s", gres.pkid, iconName, size, defaultIconPolicy.iconSize);
auto info = iconsStored[size];
lastIconName = info.fname;
if (storeIcon (cpt, gres, cptMediaPath, info.pkg, lastIconName, defaultIconPolicy))
return true;
}
}
// if we are here, we either didn't find an icon, or no icon is present
// in the default 64x64px size
return false;
}
// search for the right icon iside the current package
auto success = findAndStoreXdgIcon (gres.pkg);
if (!success && !gres.isIgnored (cpt)) {
// search in all packages
success = findAndStoreXdgIcon ();
}
if (success) {
logDebug ("Icon %s - %s found in XDG dirs", gres.pkid, iconName);
// we found a valid stock icon, so set that additionally to the cached one
auto icon = new Icon;
icon.setKind (IconKind.STOCK);
icon.setName (iconName);
cpt.addIcon (icon);
return true;
} else {
logDebug ("Icon %s - %s not found in required size(s) in XDG dirs", gres.pkid, iconName);
if ((lastIconName !is null) && (!iconAllowed (lastIconName))) {
gres.addHint (cpt.getId, "icon-format-unsupported", ["icon_fname": baseName (lastIconName)]);
return false;
}
gres.addHint (cpt.getId, "icon-not-found", ["icon_fname": iconName]);
return false;
}
}
}
} // end of IconHandler class
unittest
{
writeln ("TEST: ", "IconHandler");
auto hicolorThemeIndex = getDataPath ("hicolor-theme-index.theme");
ubyte[] indexData;
auto f = File (hicolorThemeIndex, "r");
while (!f.eof) {
char[GENERIC_BUFFER_SIZE] buf;
indexData ~= f.rawRead (buf);
}
auto theme = new Theme ("hicolor", indexData);
foreach (fname; theme.matchingIconFilenames ("accessories-calculator", ImageSize (48))) {
bool valid = false;
if (fname.startsWith ("/usr/share/icons/hicolor/48x48/"))
valid = true;
if (fname.startsWith ("/usr/share/icons/hicolor/scalable/"))
valid = true;
assert (valid);
if ((valid) && (IconHandler.iconAllowed (fname)))
valid = true;
else
valid = false;
if (fname.endsWith (".ico"))
assert (!valid);
else
assert (valid);
}
foreach (fname; theme.matchingIconFilenames ("accessories-text-editor", ImageSize (192))) {
if (fname.startsWith ("/usr/share/icons/hicolor/192x192/"))
continue;
if (fname.startsWith ("/usr/share/icons/hicolor/256x256/"))
continue;
if (fname.startsWith ("/usr/share/icons/hicolor/512x512/"))
continue;
if (fname.startsWith ("/usr/share/icons/hicolor/scalable/"))
continue;
assert (0);
}
}
appstream-generator-0.8.6/src/asgen/localeunit.d 0000664 0000000 0000000 00000021102 14160667064 0021703 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2018-2021 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
import std.conv : to;
import std.string : format;
import glib.PtrArray : PtrArray;
import glib.c.types : GQuark, GError, GBytes;
import ascompose.Unit : Unit;
import ascompose.c.types;
import asgen.config : Config;
import asgen.contentsstore : ContentsStore;
import asgen.backends.interfaces : Package;
private __gshared extern(C) GQuark asc_compose_error_quark ();
/**
* Special unit that wraps the global locale fetching code.
*/
public final class LocaleUnit : Unit
{
package Package[string] localeFilePkgMap;
import gobject.Type : Type;
import gobject.c.functions : g_object_get_data;
import ascompose.c.functions: asc_unit_get_type;
import ascompose.c.functions : asc_unit_get_user_data;
struct AGLocaleUnit
{
AscUnit parentInstance;
}
struct AGLocaleUnitClass
{
AscUnitClass parentClass;
}
protected AGLocaleUnit *agLocaleUnit;
protected override void* getStruct()
{
return cast(void*)gObject;
}
public synchronized static GType getType()
{
import std.algorithm : startsWith;
GType agLocaleUnitType = Type.fromName("AGLocaleUnit");
if (agLocaleUnitType == GType.INVALID) {
agLocaleUnitType = Type.registerStaticSimple(
asc_unit_get_type(),
"AGLocaleUnit",
cast(uint)AGLocaleUnitClass.sizeof,
cast(GClassInitFunc) &agLocaleUnitClassInit,
cast(uint)AGLocaleUnit.sizeof, null, cast(GTypeFlags)0);
foreach (member; __traits(derivedMembers, LocaleUnit))
{
static if (member.startsWith("_implementInterface"))
__traits(getMember, LocaleUnit, member)(agLocaleUnitType);
}
}
return agLocaleUnitType;
}
public this (ContentsStore cstore, Package[] pkgList)
{
import std.array : array;
import std.typecons : scoped;
import std.string : toStringz;
import glib.c.functions : g_strdup, g_free;
import gobject.c.functions : g_object_newv;
super (cast(AscUnit*) g_object_newv (getType (), 0, null), true);
// helper for function override callbacks
setUserData (cast(void*) this);
// convert the list into an associative array for faster lookups
Package[string] pkgMap;
foreach (ref pkg; pkgList) {
immutable pkid = pkg.id;
pkgMap[pkid] = pkg;
}
pkgMap.rehash ();
auto conf = Config.get;
if (!conf.feature.processLocale)
return; // don't load the expensive locale<->package mapping if we don't need it
// we make the assumption here that all locale for a given domain are in one package.
// otherwise this global search will get even more insane.
// the key of the map returned by getLocaleMap will therefore contain only the locale
// file basename instead of a full path
auto dbLocaleMap = cstore.getLocaleMap (array(pkgMap.byKey));
foreach (ref info; dbLocaleMap.byKeyValue) {
immutable id = info.key;
immutable pkgid = info.value;
// check if we already have a package - lookups in this HashMap are faster
// due to its smaller size and (most of the time) outweight the following additional
// lookup for the right package entity.
if (localeFilePkgMap.get (id, null) !is null)
continue;
Package pkg;
if (pkgid !is null)
pkg = pkgMap.get (pkgid, null);
if (pkg !is null)
localeFilePkgMap[id] = pkg;
}
auto contents = new PtrArray (&g_free);
foreach (ref fname; localeFilePkgMap.byKey)
contents.add (cast(void*) g_strdup (fname.toStringz));
setContents (contents);
}
extern(C)
{
static void agLocaleUnitClassInit (void* klass)
{
AscUnitClass *ascUnitClass = cast(AscUnitClass*) klass;
gobject.c.types.GObjectClass *gobjectCTypesGObjectClass = cast(gobject.c.types.GObjectClass*)klass;
ascUnitClass.open = &agLocaleUnitOpen;
ascUnitClass.close = &agLocaleUnitClose;
ascUnitClass.fileExists = &agLocaleUnitFileExists;
ascUnitClass.dirExists = &agLocaleUnitDirExists;
ascUnitClass.readData = &agLocaleUnitReadData;
}
static int agLocaleUnitOpen (AscUnit *iface, GError** err)
{
auto impl = cast(LocaleUnit) g_object_get_data (cast(GObject*)iface, "GObject".ptr);
if (impl is null)
impl = cast(LocaleUnit) asc_unit_get_user_data (iface);
return impl.open ()? 1 : 0;
}
static void agLocaleUnitClose (AscUnit* iface)
{
auto impl = cast(LocaleUnit) g_object_get_data (cast(GObject*)iface, "GObject".ptr);
if (impl is null)
impl = cast(LocaleUnit) asc_unit_get_user_data (iface);
impl.close();
}
static int agLocaleUnitFileExists (AscUnit* iface, const(char)* dirname)
{
import std.string : fromStringz;
auto impl = cast(LocaleUnit) g_object_get_data (cast(GObject*)iface, "GObject".ptr);
if (impl is null)
impl = cast(LocaleUnit) asc_unit_get_user_data (iface);
return impl.fileExists (dirname.fromStringz.to!string)? 1 : 0;
}
static int agLocaleUnitDirExists (AscUnit* iface, const(char)* dirname)
{
import std.string : fromStringz;
auto impl = cast(LocaleUnit) g_object_get_data (cast(GObject*)iface, "GObject".ptr);
if (impl is null)
impl = cast(LocaleUnit) asc_unit_get_user_data (iface);
return impl.dirExists (dirname.fromStringz.to!string)? 1 : 0;
}
static GBytes *agLocaleUnitReadData (AscUnit* iface, const(char)* filename, GError** err)
{
auto impl = cast(LocaleUnit) g_object_get_data (cast(GObject*)iface, "GObject".ptr);
if (impl is null)
impl = cast(LocaleUnit) asc_unit_get_user_data (iface);
return impl.c_readData (filename, err);
}
}
public override bool open ()
{
// we already opened everything in the constructor
// (this enables us to reuse this fake unit multiple times)
return true;
}
public override void close ()
{
// noop
}
public override bool fileExists (string filename)
{
return ((filename in localeFilePkgMap) is null)? false : true;
}
public override bool dirExists (string dirname)
{
// not implemented yet, as it's not needed for locale finding (yet?)
return false;
}
package GBytes *c_readData (const(char)* filename, GError** err)
{
import glib.c.functions : g_bytes_new_take, g_memdup, g_set_error_literal;
import ascompose.c.types : ComposeError;
import ascompose.c.functions : asc_unit_get_user_data;
import std.string : toStringz, fromStringz;
const fname = filename.fromStringz.to!string;
auto pkgP = fname in localeFilePkgMap;
if (pkgP is null) {
g_set_error_literal (err,
asc_compose_error_quark,
ComposeError.FAILED,
toStringz ("File '%s' does not exist in a known package!".format(fname)));
return null;
}
const data = cast(ubyte[]) (*pkgP).getFileData (fname);
// FIXME: We should use g_memdup2 here, once we can bump the GLib version!
void *ncCopy = g_memdup (cast(void*) data.ptr, cast(uint) data.length);
return g_bytes_new_take (ncCopy, cast(size_t) data.length);
}
}
appstream-generator-0.8.6/src/asgen/logging.d 0000664 0000000 0000000 00000005061 14160667064 0021200 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2016-2021 Matthias Klumpp
*
* Licensed under the GNU Lesser General Public License Version 3
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*/
module asgen.logging;
import std.stdio;
import std.string : format;
import std.datetime;
import glib.c.types : LogLevelFlags;
private __gshared bool __verbose = false;
enum LogSeverity : string
{
DEBUG = "DEBUG",
INFO = "INFO",
WARNING = "WARNING",
ERROR = "ERROR"
}
@trusted
void logMessage (LogSeverity, string, Args...) (const LogSeverity severity, const string tmpl, const Args args)
{
auto time = Clock.currTime ();
auto timeStr = "%d-%02d-%02d %02d:%02d:%02d".format (time.year, time.month, time.day, time.hour,time.minute, time.second);
writeln (timeStr, " - ", severity, ": ", format (tmpl, args));
}
@trusted
void logDebug (string, Args...) (const string tmpl, const Args args)
{
if (__verbose)
logMessage (LogSeverity.DEBUG, tmpl, args);
}
@safe
void logInfo (string, Args...) (const string tmpl, const Args args)
{
logMessage (LogSeverity.INFO, tmpl, args);
}
@safe
void logWarning (string, Args...) (const string tmpl, const Args args)
{
logMessage (LogSeverity.WARNING, tmpl, args);
}
@safe
void logError (string, Args...) (const string tmpl, const Args args)
{
logMessage (LogSeverity.ERROR, tmpl, args);
}
private extern(C) void logGExternal (const(char)* logDomain,
LogLevelFlags logLevel,
const(char)* message, void* userData)
{
import std.string : fromStringz;
logDebug (message.fromStringz);
}
@trusted
void setVerbose (const bool enabled)
{
import glib.MessageLog : MessageLog;
__verbose = enabled;
if (__verbose) {
MessageLog.logSetHandler (null,
LogLevelFlags.LEVEL_DEBUG,
&logGExternal,
null);
}
}
appstream-generator-0.8.6/src/asgen/meson.build 0000664 0000000 0000000 00000005136 14160667064 0021552 0 ustar 00root root 0000000 0000000
conf_data = configuration_data()
conf_data.set('datadir', join_paths(get_option('prefix'), get_option('datadir'), 'appstream'))
conf_data.set('asgen_version', asgen_version)
configure_file(input : 'defines.d.in',
output: 'defines.d',
configuration: conf_data
)
data_import_dirs = include_directories('../../data')
#
# Sources
#
asgen_sources = [
'../app.d',
'bindings/libarchive.d',
'bindings/lmdb.d',
'config.d',
'contentsstore.d',
'datainjectpkg.d',
'datastore.d',
'downloader.d',
'engine.d',
'extractor.d',
'hintregistry.d',
'iconhandler.d',
'localeunit.d',
'logging.d',
'mustache.d',
'packageunit.d',
'reportgenerator.d',
'result.d',
'utils.d',
'zarchive.d',
join_paths(build_root, 'src', 'asgen', 'defines.d'),
]
backend_sources = [
'backends/interfaces.d',
'backends/dummy/package.d',
'backends/dummy/dummypkg.d',
'backends/dummy/pkgindex.d',
'backends/alpinelinux/package.d',
'backends/alpinelinux/apkindexutils.d',
'backends/alpinelinux/apkpkg.d',
'backends/alpinelinux/apkpkgindex.d',
'backends/archlinux/package.d',
'backends/archlinux/alpkg.d',
'backends/archlinux/alpkgindex.d',
'backends/archlinux/listfile.d',
'backends/debian/package.d',
'backends/debian/debpkg.d',
'backends/debian/debpkgindex.d',
'backends/debian/tagfile.d',
'backends/debian/debutils.d',
'backends/ubuntu/package.d',
'backends/ubuntu/ubupkg.d',
'backends/ubuntu/ubupkgindex.d',
'backends/rpmmd/package.d',
'backends/rpmmd/rpmpkg.d',
'backends/rpmmd/rpmpkgindex.d',
]
#
# Build and test
#
asgen_exe = executable('appstream-generator',
[asgen_sources,
backend_sources],
include_directories: [src_dir,
gir_bind_dir],
dependencies: [glibd_dep,
appstream_dep,
ascompose_dep,
lmdb_dep,
archive_dep,
curl_dep],
link_with: [girbind_lib],
d_import_dirs: [data_import_dirs],
install: true
)
asgen_test_exe = executable('asgen_test',
[asgen_sources,
backend_sources],
include_directories: [src_dir,
gir_bind_dir],
dependencies: [glibd_dep,
appstream_dep,
ascompose_dep,
lmdb_dep,
archive_dep,
curl_dep],
link_with: [girbind_lib],
d_import_dirs: [data_import_dirs],
d_unittest: true
)
test('asgen_tests',
asgen_test_exe,
workdir: meson.source_root())
appstream-generator-0.8.6/src/asgen/mustache.d 0000664 0000000 0000000 00000127566 14160667064 0021402 0 ustar 00root root 0000000 0000000 /*
* Copyright (C) 2011 Masahiro Nakagawa
*
* Licensed under the GNU Lesser General Public License Version 3
* or Boost Software License 1.0.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the license, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this software. If not, see .
*
* Mustache template engine for D
*
* Implemented according to [mustach(5)](http://mustache.github.com/mustache.5.html)
* BSL-1.0 licensed source from https://github.com/repeatedly/mustache-d
*/
module mustache;
import std.algorithm : all;
import std.array; // empty, back, popBack, appender
import std.conv; // to
import std.datetime; // SysTime (I think std.file should import std.datetime as public)
import std.file; // read, timeLastModified
import std.path; // buildPath
import std.range; // isOutputRange
import std.string; // strip, chomp, stripLeft
import std.traits; // isSomeString, isAssociativeArray
static import std.ascii; // isWhite;
version(unittest) import core.thread;
/**
* Exception for Mustache
*/
class MustacheException : Exception
{
this(string messaage)
{
super(messaage);
}
}
/**
* Core implementation of Mustache
*
* $(D_PARAM String) parameter means a string type to render.
*
* Example:
* -----
* alias MustacheEngine!(string) Mustache;
*
* Mustache mustache;
* auto context = new Mustache.Context;
*
* context["name"] = "Chris";
* context["value"] = 10000;
* context["taxed_value"] = 10000 - (10000 * 0.4);
* context.useSection("in_ca");
*
* write(mustache.render("sample", context));
* -----
* sample.mustache:
* -----
* Hello {{name}}
* You have just won ${{value}}!
* {{#in_ca}}
* Well, ${{taxed_value}}, after taxes.
* {{/in_ca}}
* -----
* Output:
* -----
* Hello Chris
* You have just won $10000!
* Well, $6000, after taxes.
* -----
*/
struct MustacheEngine(String = string)
if (isSomeString!(String))
{
static assert(!is(String == wstring), "wstring is unsupported. It's a buggy!");
public:
alias Handler = String delegate(String);
alias FindPath = string delegate(string);
/**
* Cache level for compile result
*/
static enum CacheLevel
{
no, /// No caching
check, /// Caches compiled result and checks the freshness of template
once /// Caches compiled result but not check the freshness of template
}
/**
* Options for rendering
*/
static struct Option
{
string ext = "mustache"; /// template file extenstion
string path = "."; /// root path for template file searching
FindPath findPath; /// dynamically finds the path for a name
CacheLevel level = CacheLevel.check; /// See CacheLevel
Handler handler; /// Callback handler for unknown name
}
/**
* Mustache context for setting values
*
* Variable:
* -----
* //{{name}} to "Chris"
* context["name"] = "Chirs"
* -----
*
* Lists section("addSubContext" name is drived from ctemplate's API):
* -----
* //{{#repo}}
* //{{name}}
* //{{/repo}}
* // to
* //resque
* //hub
* //rip
* foreach (name; ["resque", "hub", "rip"]) {
* auto sub = context.addSubContext("repo");
* sub["name"] = name;
* }
* -----
*
* Variable section:
* -----
* //{{#person?}}Hi {{name}}{{/person?}} to "Hi Jon"
* context["person?"] = ["name" : "Jon"];
* -----
*
* Lambdas section:
* -----
* //{{#wrapped}}awesome{{/wrapped}} to "awesome"
* context["Wrapped"] = (string str) { return "" ~ str ~ ""; };
* -----
*
* Inverted section:
* -----
* //{{#repo}}{{name}}{{/repo}}
* //{{^repo}}No repos :({{/repo}}
* // to
* //No repos :(
* context["foo"] = "bar"; // not set to "repo"
* -----
*/
static final class Context
{
private:
enum SectionType
{
nil, use, var, func, list
}
struct Section
{
SectionType type;
union
{
String[String] var;
String delegate(String) func; // func type is String delegate(String) delegate()?
Context[] list;
}
@trusted
this(bool u) nothrow
{
type = SectionType.use;
}
@trusted
this(String[String] v) nothrow
{
type = SectionType.var;
var = v;
}
@trusted
this(String delegate(String) f) nothrow
{
type = SectionType.func;
func = f;
}
@trusted
this(Context c) nothrow
{
type = SectionType.list;
list = [c];
}
@trusted
this(Context[] c) nothrow
{
type = SectionType.list;
list = c;
}
/* nothrow : AA's length is not nothrow */
@trusted @property
bool empty() const
{
final switch (type) {
case SectionType.nil:
return true;
case SectionType.use:
return false;
case SectionType.var:
return !var.length; // Why?
case SectionType.func:
return func is null;
case SectionType.list:
return !list.length;
}
}
/* Convenience function */
@safe @property
static Section nil() nothrow
{
Section result;
result.type = SectionType.nil;
return result;
}
}
const Context parent;
String[String] variables;
Section[String] sections;
public:
@safe
this(in Context context = null) nothrow
{
parent = context;
}
/**
* Gets $(D_PARAM key)'s value. This method does not search Section.
*
* Params:
* key = key string to search
*
* Returns:
* a $(D_PARAM key) associated value.
*
* Throws:
* a RangeError if $(D_PARAM key) does not exist.
*/
@safe
String opIndex(in String key) const nothrow
{
return variables[key];
}
/**
* Assigns $(D_PARAM value)(automatically convert to String) to $(D_PARAM key) field.
*
* If you try to assign associative array or delegate,
* This method assigns $(D_PARAM value) as Section.
*
* Arrays of Contexts are accepted, too.
*
* Params:
* value = some type value to assign
* key = key string to assign
*/
@trusted
void opIndexAssign(T)(T value, in String key)
{
static if (isAssociativeArray!(T))
{
static if (is(T V : V[K], K : String))
{
String[String] aa;
static if (is(V == String))
aa = value;
else
foreach (k, v; value) aa[k] = to!String(v);
sections[key] = Section(aa);
}
else static assert(false, "Non-supported Associative Array type");
}
else static if (isCallable!T)
{
import std.functional : toDelegate;
auto v = toDelegate(value);
static if (is(typeof(v) D == S delegate(S), S : String))
sections[key] = Section(v);
else static assert(false, "Non-supported delegate type");
}
else static if (isArray!T && !isSomeString!T)
{
static if (is(T : Context[]))
sections[key] = Section(value);
else static assert(false, "Non-supported array type");
}
else
{
variables[key] = to!String(value);
}
}
/**
* Enable $(D_PARAM key)'s section.
*
* Params:
* key = key string to enable
*
* NOTE:
* I don't like this method, but D's typing can't well-handle Ruby's typing.
*/
@safe
void useSection(in String key)
{
sections[key] = Section(true);
}
/**
* Adds new context to $(D_PARAM key)'s section. This method overwrites with
* list type if you already assigned other type to $(D_PARAM key)'s section.
*
* Params:
* key = key string to add
* size = reserve size for avoiding reallocation
*
* Returns:
* new Context object that added to $(D_PARAM key) section list.
*/
@trusted
Context addSubContext(in String key, lazy size_t size = 1)
{
auto c = new Context(this);
auto p = key in sections;
if (!p || p.type != SectionType.list) {
sections[key] = Section(c);
sections[key].list.reserve(size);
} else {
sections[key].list ~= c;
}
return c;
}
private:
/*
* Fetches $(D_PARAM)'s value. This method follows parent context.
*
* Params:
* key = key string to fetch
*
* Returns:
* a $(D_PARAM key) associated value. null if key does not exist.
*/
@trusted
String fetch(in String[] key, lazy Handler handler = null) const
{
assert(key.length > 0);
if (key.length == 1) {
auto result = key[0] in variables;
if (result !is null)
return *result;
if (parent !is null)
return parent.fetch(key, handler);
} else {
auto contexts = fetchList(key[0..$-1]);
foreach (c; contexts) {
auto result = key[$-1] in c.variables;
if (result !is null)
return *result;
}
}
return handler is null ? null : handler()(keyToString(key));
}
@trusted
const(Section) fetchSection()(in String[] key) const /* nothrow */
{
assert(key.length > 0);
// Ascend context tree to find the key's beginning
auto currentSection = key[0] in sections;
if (currentSection is null) {
if (parent is null)
return Section.nil;
return parent.fetchSection(key);
}
// Decend context tree to match the rest of the key
size_t keyIndex = 0;
while (currentSection) {
// Matched the entire key?
if (keyIndex == key.length.to!long - 1)
return currentSection.empty ? Section.nil : *currentSection;
if (currentSection.type != SectionType.list)
return Section.nil; // Can't decend any further
// Find next part of key
keyIndex++;
foreach (c; currentSection.list)
{
currentSection = key[keyIndex] in c.sections;
if (currentSection)
break;
}
}
return Section.nil;
}
@trusted
const(Result) fetchSection(Result, SectionType type, string name)(in String[] key) const /* nothrow */
{
auto result = fetchSection(key);
if (result.type == type)
return result.empty ? null : mixin("result." ~ to!string(type));
return null;
}
alias fetchVar = fetchSection!(String[String], SectionType.var, "Var");
alias fetchList = fetchSection!(Context[], SectionType.list, "List");
alias fetchFunc = fetchSection!(String delegate(String), SectionType.func, "Func");
}
unittest
{
Context context = new Context();
context["name"] = "Red Bull";
assert(context["name"] == "Red Bull");
context["price"] = 275;
assert(context["price"] == "275");
{ // list
foreach (i; 100..105) {
auto sub = context.addSubContext("sub");
sub["num"] = i;
foreach (b; [true, false]) {
auto subsub = sub.addSubContext("subsub");
subsub["To be or not to be"] = b;
}
}
foreach (i, sub; context.fetchList(["sub"])) {
assert(sub.fetch(["name"]) == "Red Bull");
assert(sub["num"] == to!String(i + 100));
foreach (j, subsub; sub.fetchList(["subsub"])) {
assert(subsub.fetch(["price"]) == to!String(275));
assert(subsub["To be or not to be"] == to!String(j == 0));
}
}
}
{ // variable
String[String] aa = ["name" : "Ritsu"];
context["Value"] = aa;
assert(context.fetchVar(["Value"]) == cast(const)aa);
}
{ // func
auto func = function (String str) { return "" ~ str ~ ""; };
context["Wrapped"] = func;
assert(context.fetchFunc(["Wrapped"])("Ritsu") == func("Ritsu"));
}
{ // handler
Handler fixme = delegate String(String s) { assert(s=="unknown"); return "FIXME"; };
Handler error = delegate String(String s) { assert(s=="unknown"); throw new MustacheException("Unknow"); };
assert(context.fetch(["unknown"]) == "");
assert(context.fetch(["unknown"], fixme) == "FIXME");
try {
assert(context.fetch(["unknown"], error) == "");
assert(false);
} catch (MustacheException e) { }
}
{ // subcontext
auto sub = new Context();
sub["num"] = 42;
context["a"] = [sub];
auto list = context.fetchList(["a"]);
assert(list.length == 1);
foreach (i, s; list)
assert(s["num"] == to!String(42));
}
}
private:
// Internal cache
struct Cache
{
Node[] compiled;
SysTime modified;
}
Option option_;
Cache[string] caches_;
public:
@safe
this(Option option) nothrow
{
option_ = option;
}
@property @safe nothrow
{
/**
* Property for template extenstion
*/
const(string) ext() const
{
return option_.ext;
}
/// ditto
void ext(string ext)
{
option_.ext = ext;
}
/**
* Property for template searche path
*/
const(string) path() const
{
return option_.path;
}
/// ditto
void path(string path)
{
option_.path = path;
}
/**
* Property for callback to dynamically search path.
* The result of the delegate should return the full path for
* the given name.
*/
FindPath findPath() const
{
return option_.findPath;
}
/// ditto
void findPath(FindPath findPath)
{
option_.findPath = findPath;
}
/**
* Property for cache level
*/
const(CacheLevel) level() const
{
return option_.level;
}
/// ditto
void level(CacheLevel level)
{
option_.level = level;
}
/**
* Property for callback handler
*/
const(Handler) handler() const
{
return option_.handler;
}
/// ditto
void handler(Handler handler)
{
option_.handler = handler;
}
}
/**
* Clears the intenal cache.
* Useful for forcing reloads when using CacheLevel.once.
*/
@safe
void clearCache()
{
caches_ = null;
}
/**
* Renders $(D_PARAM name) template with $(D_PARAM context).
*
* This method stores compile result in memory if you set check or once CacheLevel.
*
* Params:
* name = template name without extenstion
* context = Mustache context for rendering
*
* Returns:
* rendered result.
*
* Throws:
* object.Exception if String alignment is mismatched from template file.
*/
String render()(in string name, in Context context)
{
auto sink = appender!String();
render(name, context, sink);
return sink.data;
}
/**
* OutputRange version of $(D render).
*/
void render(Sink)(in string name, in Context context, ref Sink sink)
if(isOutputRange!(Sink, String))
{
/*
* Helper for file reading
*
* Throws:
* object.Exception if alignment is mismatched.
*/
@trusted
static String readFile(string file)
{
// cast checks character encoding alignment.
return cast(String)read(file);
}
string file;
if (option_.findPath) {
file = option_.findPath(name);
} else {
file = buildPath(option_.path, name ~ "." ~ option_.ext);
}
Node[] nodes;
final switch (option_.level) {
case CacheLevel.no:
nodes = compile(readFile(file));
break;
case CacheLevel.check:
auto t = timeLastModified(file);
auto p = file in caches_;
if (!p || t > p.modified)
caches_[file] = Cache(compile(readFile(file)), t);
nodes = caches_[file].compiled;
break;
case CacheLevel.once:
if (file !in caches_)
caches_[file] = Cache(compile(readFile(file)), SysTime.min);
nodes = caches_[file].compiled;
break;
}
renderImpl(nodes, context, sink);
}
/**
* string version of $(D render).
*/
String renderString()(in String src, in Context context)
{
auto sink = appender!String();
renderString(src, context, sink);
return sink.data;
}
/**
* string/OutputRange version of $(D render).
*/
void renderString(Sink)(in String src, in Context context, ref Sink sink)
if(isOutputRange!(Sink, String))
{
renderImpl(compile(src), context, sink);
}
private:
/*
* Implemention of render function.
*/
void renderImpl(Sink)(in Node[] nodes, in Context context, ref Sink sink)
if(isOutputRange!(Sink, String))
{
// helper for HTML escape(original function from std.xml.encode)
static void encode(in String text, ref Sink sink)
{
size_t index;
foreach (i, c; text) {
String temp;
switch (c) {
case '&': temp = "&"; break;
case '"': temp = """; break;
case '<': temp = "<"; break;
case '>': temp = ">"; break;
default: continue;
}
sink.put(text[index .. i]);
sink.put(temp);
index = i + 1;
}
sink.put(text[index .. $]);
}
foreach (ref node; nodes) {
final switch (node.type) {
case NodeType.text:
sink.put(node.text);
break;
case NodeType.var:
auto value = context.fetch(node.key, option_.handler);
if (value)
{
if(node.flag)
sink.put(value);
else
encode(value, sink);
}
break;
case NodeType.section:
auto section = context.fetchSection(node.key);
final switch (section.type) {
case Context.SectionType.nil:
if (node.flag)
renderImpl(node.childs, context, sink);
break;
case Context.SectionType.use:
if (!node.flag)
renderImpl(node.childs, context, sink);
break;
case Context.SectionType.var:
auto var = section.var;
auto sub = new Context(context);
foreach (k, v; var)
sub[k] = v;
renderImpl(node.childs, sub, sink);
break;
case Context.SectionType.func:
auto func = section.func;
renderImpl(compile(func(node.source)), context, sink);
break;
case Context.SectionType.list:
auto list = section.list;
if (!node.flag) {
foreach (sub; list)
renderImpl(node.childs, sub, sink);
}
break;
}
break;
case NodeType.partial:
render(to!string(node.key.front), context, sink);
break;
}
}
}
unittest
{
MustacheEngine!(String) m;
auto render = (String str, Context c) => m.renderString(str, c);
{ // var
auto context = new Context;
context["name"] = "Ritsu & Mio";
assert(render("Hello {{name}}", context) == "Hello Ritsu & Mio");
assert(render("Hello {{&name}}", context) == "Hello Ritsu & Mio");
assert(render("Hello {{{name}}}", context) == "Hello Ritsu & Mio");
}
{ // var with handler
auto context = new Context;
context["name"] = "Ritsu & Mio";
m.handler = delegate String(String s) { assert(s=="unknown"); return "FIXME"; };
assert(render("Hello {{unknown}}", context) == "Hello FIXME");
m.handler = delegate String(String s) { assert(s=="unknown"); throw new MustacheException("Unknow"); };
try {
assert(render("Hello {{&unknown}}", context) == "Hello Ritsu & Mio");
assert(false);
} catch (MustacheException e) {}
m.handler = null;
}
{ // list section
auto context = new Context;
foreach (name; ["resque", "hub", "rip"]) {
auto sub = context.addSubContext("repo");
sub["name"] = name;
}
assert(render("{{#repo}}\n {{name}}\n{{/repo}}", context) ==
" resque\n hub\n rip\n");
}
{ // var section
auto context = new Context;
String[String] aa = ["name" : "Ritsu"];
context["person?"] = aa;
assert(render("{{#person?}} Hi {{name}}!\n{{/person?}}", context) ==
" Hi Ritsu!\n");
}
{ // inverted section
{
String temp = "{{#repo}}\n{{name}}\n{{/repo}}\n{{^repo}}\nNo repos :(\n{{/repo}}\n";
auto context = new Context;
assert(render(temp, context) == "\nNo repos :(\n");
String[String] aa;
context["person?"] = aa;
assert(render(temp, context) == "\nNo repos :(\n");
}
{
auto temp = "{{^section}}This shouldn't be seen.{{/section}}";
auto context = new Context;
context.addSubContext("section")["foo"] = "bar";
assert(render(temp, context).empty);
}
}
{ // comment
auto context = new Context;
assert(render("
Today{{! ignore me }}.
", context) == "
Today.
");
}
{ // partial
std.file.write("user.mustache", to!String("{{name}}"));
scope(exit) std.file.remove("user.mustache");
auto context = new Context;
foreach (name; ["Ritsu", "Mio"]) {
auto sub = context.addSubContext("names");
sub["name"] = name;
}
assert(render("
